2024-11-16 10:14:02,565 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-16 10:14:02,578 main DEBUG Took 0.010919 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-16 10:14:02,578 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-16 10:14:02,578 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-16 10:14:02,579 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-16 10:14:02,580 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:14:02,589 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-16 10:14:02,602 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:14:02,603 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:14:02,604 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:14:02,605 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:14:02,605 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:14:02,606 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:14:02,607 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:14:02,608 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:14:02,608 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:14:02,609 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:14:02,610 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:14:02,611 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:14:02,611 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:14:02,612 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:14:02,613 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:14:02,613 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:14:02,614 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:14:02,614 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:14:02,615 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:14:02,615 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:14:02,616 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:14:02,616 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:14:02,617 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:14:02,617 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:14:02,618 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:14:02,618 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-16 10:14:02,620 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:14:02,622 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-16 10:14:02,624 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-16 10:14:02,625 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-16 10:14:02,627 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-16 10:14:02,627 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-16 10:14:02,637 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-16 10:14:02,640 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-16 10:14:02,643 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-16 10:14:02,643 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-16 10:14:02,644 main DEBUG createAppenders(={Console}) 2024-11-16 10:14:02,645 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-16 10:14:02,645 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-16 10:14:02,645 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-16 10:14:02,646 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-16 10:14:02,646 main DEBUG OutputStream closed 2024-11-16 10:14:02,647 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-16 10:14:02,647 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-16 10:14:02,648 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-16 10:14:02,727 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-16 10:14:02,729 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-16 10:14:02,730 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-16 10:14:02,731 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-16 10:14:02,731 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-16 10:14:02,732 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-16 10:14:02,732 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-16 10:14:02,732 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-16 10:14:02,733 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-16 10:14:02,733 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-16 10:14:02,733 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-16 10:14:02,733 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-16 10:14:02,734 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-16 10:14:02,734 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-16 10:14:02,734 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-16 10:14:02,734 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-16 10:14:02,735 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-16 10:14:02,736 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-16 10:14:02,738 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16 10:14:02,739 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-16 10:14:02,739 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-16 10:14:02,740 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-16T10:14:02,954 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569 2024-11-16 10:14:02,957 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-16 10:14:02,957 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16T10:14:02,967 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-16T10:14:02,988 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-16T10:14:03,008 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/cluster_bb61cbda-8873-24b8-881e-9662d491e561, deleteOnExit=true 2024-11-16T10:14:03,009 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-16T10:14:03,010 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/test.cache.data in system properties and HBase conf 2024-11-16T10:14:03,011 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/hadoop.tmp.dir in system properties and HBase conf 2024-11-16T10:14:03,012 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/hadoop.log.dir in system properties and HBase conf 2024-11-16T10:14:03,013 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-16T10:14:03,014 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-16T10:14:03,014 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-16T10:14:03,109 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-16T10:14:03,194 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T10:14:03,198 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-16T10:14:03,199 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-16T10:14:03,199 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-16T10:14:03,200 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T10:14:03,200 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-16T10:14:03,201 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-16T10:14:03,201 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T10:14:03,202 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T10:14:03,202 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-16T10:14:03,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/nfs.dump.dir in system properties and HBase conf 2024-11-16T10:14:03,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/java.io.tmpdir in system properties and HBase conf 2024-11-16T10:14:03,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T10:14:03,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-16T10:14:03,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-16T10:14:04,277 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-16T10:14:04,354 INFO [Time-limited test {}] log.Log(170): Logging initialized @2472ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-16T10:14:04,428 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T10:14:04,493 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T10:14:04,517 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T10:14:04,517 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T10:14:04,518 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-16T10:14:04,530 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T10:14:04,533 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/hadoop.log.dir/,AVAILABLE} 2024-11-16T10:14:04,535 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T10:14:04,713 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/java.io.tmpdir/jetty-localhost-44085-hadoop-hdfs-3_4_1-tests_jar-_-any-1151479184015019818/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T10:14:04,719 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:44085} 2024-11-16T10:14:04,720 INFO [Time-limited test {}] server.Server(415): Started @2838ms 2024-11-16T10:14:05,264 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T10:14:05,272 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T10:14:05,273 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T10:14:05,273 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T10:14:05,273 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T10:14:05,274 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/hadoop.log.dir/,AVAILABLE} 2024-11-16T10:14:05,275 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T10:14:05,375 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/java.io.tmpdir/jetty-localhost-33269-hadoop-hdfs-3_4_1-tests_jar-_-any-16456216577603050009/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T10:14:05,376 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:33269} 2024-11-16T10:14:05,376 INFO [Time-limited test {}] server.Server(415): Started @3495ms 2024-11-16T10:14:05,426 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T10:14:06,892 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/cluster_bb61cbda-8873-24b8-881e-9662d491e561/data/data1/current/BP-1732300688-172.17.0.2-1731752043743/current, will proceed with Du for space computation calculation, 2024-11-16T10:14:06,892 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/cluster_bb61cbda-8873-24b8-881e-9662d491e561/data/data2/current/BP-1732300688-172.17.0.2-1731752043743/current, will proceed with Du for space computation calculation, 2024-11-16T10:14:06,934 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T10:14:06,985 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x18528c6ce4e36dd3 with lease ID 0x41ede3ec1f9eb6e5: Processing first storage report for DS-daab0977-6a66-4666-bc1e-a135402836ed from datanode DatanodeRegistration(127.0.0.1:44191, datanodeUuid=c61bbee8-8e42-49b2-9ad9-00f98f01cef0, infoPort=40949, infoSecurePort=0, ipcPort=32941, storageInfo=lv=-57;cid=testClusterID;nsid=1041107819;c=1731752043743) 2024-11-16T10:14:06,986 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x18528c6ce4e36dd3 with lease ID 0x41ede3ec1f9eb6e5: from storage DS-daab0977-6a66-4666-bc1e-a135402836ed node DatanodeRegistration(127.0.0.1:44191, datanodeUuid=c61bbee8-8e42-49b2-9ad9-00f98f01cef0, infoPort=40949, infoSecurePort=0, ipcPort=32941, storageInfo=lv=-57;cid=testClusterID;nsid=1041107819;c=1731752043743), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-16T10:14:06,987 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x18528c6ce4e36dd3 with lease ID 0x41ede3ec1f9eb6e5: Processing first storage report for DS-48716b36-ad08-4e9a-b0cf-af905a11b26c from datanode DatanodeRegistration(127.0.0.1:44191, datanodeUuid=c61bbee8-8e42-49b2-9ad9-00f98f01cef0, infoPort=40949, infoSecurePort=0, ipcPort=32941, storageInfo=lv=-57;cid=testClusterID;nsid=1041107819;c=1731752043743) 2024-11-16T10:14:06,987 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x18528c6ce4e36dd3 with lease ID 0x41ede3ec1f9eb6e5: from storage DS-48716b36-ad08-4e9a-b0cf-af905a11b26c node DatanodeRegistration(127.0.0.1:44191, datanodeUuid=c61bbee8-8e42-49b2-9ad9-00f98f01cef0, infoPort=40949, infoSecurePort=0, ipcPort=32941, storageInfo=lv=-57;cid=testClusterID;nsid=1041107819;c=1731752043743), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T10:14:07,032 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569 2024-11-16T10:14:07,112 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/cluster_bb61cbda-8873-24b8-881e-9662d491e561/zookeeper_0, clientPort=64641, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/cluster_bb61cbda-8873-24b8-881e-9662d491e561/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/cluster_bb61cbda-8873-24b8-881e-9662d491e561/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-16T10:14:07,121 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64641 2024-11-16T10:14:07,131 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T10:14:07,133 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T10:14:07,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741825_1001 (size=7) 2024-11-16T10:14:07,745 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0 with version=8 2024-11-16T10:14:07,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/hbase-staging 2024-11-16T10:14:07,835 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-16T10:14:08,042 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b16ff38458c1:0 server-side Connection retries=45 2024-11-16T10:14:08,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T10:14:08,051 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T10:14:08,055 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T10:14:08,056 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T10:14:08,056 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T10:14:08,200 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T10:14:08,252 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-16T10:14:08,260 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-16T10:14:08,263 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T10:14:08,291 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4801 (auto-detected) 2024-11-16T10:14:08,292 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-16T10:14:08,314 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45949 2024-11-16T10:14:08,335 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45949 connecting to ZooKeeper ensemble=127.0.0.1:64641 2024-11-16T10:14:08,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:459490x0, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T10:14:08,415 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45949-0x10143232da10000 connected 2024-11-16T10:14:08,813 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T10:14:08,814 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T10:14:08,822 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T10:14:08,826 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0, hbase.cluster.distributed=false 2024-11-16T10:14:08,846 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T10:14:08,850 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45949 2024-11-16T10:14:08,851 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45949 2024-11-16T10:14:08,851 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45949 2024-11-16T10:14:08,852 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45949 2024-11-16T10:14:08,852 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45949 2024-11-16T10:14:08,943 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b16ff38458c1:0 server-side Connection retries=45 2024-11-16T10:14:08,944 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T10:14:08,945 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T10:14:08,945 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T10:14:08,945 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T10:14:08,945 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T10:14:08,947 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T10:14:08,950 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T10:14:08,951 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45417 2024-11-16T10:14:08,953 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45417 connecting to ZooKeeper ensemble=127.0.0.1:64641 2024-11-16T10:14:08,954 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T10:14:08,957 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T10:14:09,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:454170x0, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T10:14:09,020 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45417-0x10143232da10001 connected 2024-11-16T10:14:09,020 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45417-0x10143232da10001, quorum=127.0.0.1:64641, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T10:14:09,026 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T10:14:09,035 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T10:14:09,038 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45417-0x10143232da10001, quorum=127.0.0.1:64641, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T10:14:09,044 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45417-0x10143232da10001, quorum=127.0.0.1:64641, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T10:14:09,045 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45417 2024-11-16T10:14:09,046 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45417 2024-11-16T10:14:09,046 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45417 2024-11-16T10:14:09,047 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45417 2024-11-16T10:14:09,047 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45417 2024-11-16T10:14:09,064 DEBUG [M:0;b16ff38458c1:45949 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b16ff38458c1:45949 2024-11-16T10:14:09,065 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b16ff38458c1,45949,1731752047888 2024-11-16T10:14:09,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45417-0x10143232da10001, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T10:14:09,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T10:14:09,108 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b16ff38458c1,45949,1731752047888 2024-11-16T10:14:09,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45417-0x10143232da10001, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T10:14:09,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:14:09,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45417-0x10143232da10001, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:14:09,174 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T10:14:09,176 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b16ff38458c1,45949,1731752047888 from backup master directory 2024-11-16T10:14:09,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b16ff38458c1,45949,1731752047888 2024-11-16T10:14:09,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45417-0x10143232da10001, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T10:14:09,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T10:14:09,238 WARN [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T10:14:09,239 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b16ff38458c1,45949,1731752047888 2024-11-16T10:14:09,243 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-16T10:14:09,245 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-16T10:14:09,296 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/hbase.id] with ID: f4ee6ab6-3ad8-4552-8e23-aafe5d412c59 2024-11-16T10:14:09,297 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/.tmp/hbase.id 2024-11-16T10:14:09,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741826_1002 (size=42) 2024-11-16T10:14:09,708 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/.tmp/hbase.id]:[hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/hbase.id] 2024-11-16T10:14:09,755 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T10:14:09,759 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T10:14:09,776 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-11-16T10:14:09,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:14:09,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45417-0x10143232da10001, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:14:09,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741827_1003 (size=196) 2024-11-16T10:14:10,265 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T10:14:10,268 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T10:14:10,282 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:10,286 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T10:14:10,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741828_1004 (size=1189) 2024-11-16T10:14:10,738 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/data/master/store 2024-11-16T10:14:10,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741829_1005 (size=34) 2024-11-16T10:14:11,171 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-16T10:14:11,177 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:14:11,180 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T10:14:11,180 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T10:14:11,181 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T10:14:11,183 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T10:14:11,183 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T10:14:11,183 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T10:14:11,189 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731752051180Disabling compacts and flushes for region at 1731752051180Disabling writes for close at 1731752051183 (+3 ms)Writing region close event to WAL at 1731752051183Closed at 1731752051183 2024-11-16T10:14:11,191 WARN [master/b16ff38458c1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/data/master/store/.initializing 2024-11-16T10:14:11,191 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/WALs/b16ff38458c1,45949,1731752047888 2024-11-16T10:14:11,200 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T10:14:11,219 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b16ff38458c1%2C45949%2C1731752047888, suffix=, logDir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/WALs/b16ff38458c1,45949,1731752047888, archiveDir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/oldWALs, maxLogs=10 2024-11-16T10:14:11,250 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/WALs/b16ff38458c1,45949,1731752047888/b16ff38458c1%2C45949%2C1731752047888.1731752051225, exclude list is [], retry=0 2024-11-16T10:14:11,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44191,DS-daab0977-6a66-4666-bc1e-a135402836ed,DISK] 2024-11-16T10:14:11,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-16T10:14:11,299 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/WALs/b16ff38458c1,45949,1731752047888/b16ff38458c1%2C45949%2C1731752047888.1731752051225 2024-11-16T10:14:11,300 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40949:40949)] 2024-11-16T10:14:11,301 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:14:11,301 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:14:11,304 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:14:11,304 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:14:11,335 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:14:11,368 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T10:14:11,373 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:11,377 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T10:14:11,378 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:14:11,382 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T10:14:11,382 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:11,384 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:14:11,384 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:14:11,388 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T10:14:11,388 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:11,390 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:14:11,390 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:14:11,394 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T10:14:11,394 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:11,395 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:14:11,396 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:14:11,400 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:14:11,401 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:14:11,406 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:14:11,407 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:14:11,410 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T10:14:11,414 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:14:11,418 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T10:14:11,419 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65434415, jitterRate=-0.024951234459877014}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T10:14:11,425 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731752051315Initializing all the Stores at 1731752051317 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731752051317Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752051318 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752051318Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752051318Cleaning up temporary data from old regions at 1731752051407 (+89 ms)Region opened successfully at 1731752051425 (+18 ms) 2024-11-16T10:14:11,426 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T10:14:11,455 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a46b8b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b16ff38458c1/172.17.0.2:0 2024-11-16T10:14:11,482 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T10:14:11,492 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T10:14:11,492 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T10:14:11,494 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T10:14:11,495 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-16T10:14:11,500 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-16T10:14:11,500 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T10:14:11,524 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T10:14:11,533 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T10:14:11,561 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T10:14:11,563 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T10:14:11,564 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T10:14:11,571 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T10:14:11,574 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T10:14:11,578 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T10:14:11,588 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T10:14:11,591 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T10:14:11,603 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T10:14:11,627 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T10:14:11,641 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T10:14:11,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45417-0x10143232da10001, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T10:14:11,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T10:14:11,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45417-0x10143232da10001, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:14:11,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:14:11,665 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b16ff38458c1,45949,1731752047888, sessionid=0x10143232da10000, setting cluster-up flag (Was=false) 2024-11-16T10:14:11,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45417-0x10143232da10001, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:14:11,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:14:11,729 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T10:14:11,733 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b16ff38458c1,45949,1731752047888 2024-11-16T10:14:11,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:14:11,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45417-0x10143232da10001, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:14:11,792 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T10:14:11,796 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b16ff38458c1,45949,1731752047888 2024-11-16T10:14:11,805 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T10:14:11,853 INFO [RS:0;b16ff38458c1:45417 {}] regionserver.HRegionServer(746): ClusterId : f4ee6ab6-3ad8-4552-8e23-aafe5d412c59 2024-11-16T10:14:11,856 DEBUG [RS:0;b16ff38458c1:45417 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T10:14:11,868 DEBUG [RS:0;b16ff38458c1:45417 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T10:14:11,868 DEBUG [RS:0;b16ff38458c1:45417 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T10:14:11,868 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T10:14:11,877 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T10:14:11,878 DEBUG [RS:0;b16ff38458c1:45417 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T10:14:11,878 DEBUG [RS:0;b16ff38458c1:45417 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63a8d4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b16ff38458c1/172.17.0.2:0 2024-11-16T10:14:11,883 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-16T10:14:11,888 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b16ff38458c1,45949,1731752047888 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T10:14:11,895 DEBUG [RS:0;b16ff38458c1:45417 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b16ff38458c1:45417 2024-11-16T10:14:11,896 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b16ff38458c1:0, corePoolSize=5, maxPoolSize=5 2024-11-16T10:14:11,896 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b16ff38458c1:0, corePoolSize=5, maxPoolSize=5 2024-11-16T10:14:11,896 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b16ff38458c1:0, corePoolSize=5, maxPoolSize=5 2024-11-16T10:14:11,897 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b16ff38458c1:0, corePoolSize=5, maxPoolSize=5 2024-11-16T10:14:11,897 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b16ff38458c1:0, corePoolSize=10, maxPoolSize=10 2024-11-16T10:14:11,897 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:14:11,897 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b16ff38458c1:0, corePoolSize=2, maxPoolSize=2 2024-11-16T10:14:11,898 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:14:11,898 INFO [RS:0;b16ff38458c1:45417 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T10:14:11,898 INFO [RS:0;b16ff38458c1:45417 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T10:14:11,899 DEBUG [RS:0;b16ff38458c1:45417 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T10:14:11,901 INFO [RS:0;b16ff38458c1:45417 {}] regionserver.HRegionServer(2659): reportForDuty to master=b16ff38458c1,45949,1731752047888 with port=45417, startcode=1731752048912 2024-11-16T10:14:11,908 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731752081908 2024-11-16T10:14:11,908 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T10:14:11,909 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T10:14:11,911 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T10:14:11,912 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T10:14:11,915 DEBUG [RS:0;b16ff38458c1:45417 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T10:14:11,916 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:11,916 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T10:14:11,917 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T10:14:11,916 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T10:14:11,917 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T10:14:11,917 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T10:14:11,918 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T10:14:11,926 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T10:14:11,927 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T10:14:11,928 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T10:14:11,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741831_1007 (size=1321) 2024-11-16T10:14:11,933 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T10:14:11,933 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T10:14:11,935 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b16ff38458c1:0:becomeActiveMaster-HFileCleaner.large.0-1731752051935,5,FailOnTimeoutGroup] 2024-11-16T10:14:11,936 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b16ff38458c1:0:becomeActiveMaster-HFileCleaner.small.0-1731752051935,5,FailOnTimeoutGroup] 2024-11-16T10:14:11,936 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T10:14:11,936 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T10:14:11,938 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T10:14:11,938 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T10:14:11,955 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37941, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T10:14:11,961 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45949 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b16ff38458c1,45417,1731752048912 2024-11-16T10:14:11,963 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45949 {}] master.ServerManager(517): Registering regionserver=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:11,977 DEBUG [RS:0;b16ff38458c1:45417 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0 2024-11-16T10:14:11,977 DEBUG [RS:0;b16ff38458c1:45417 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34187 2024-11-16T10:14:11,977 DEBUG [RS:0;b16ff38458c1:45417 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T10:14:11,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T10:14:11,993 DEBUG [RS:0;b16ff38458c1:45417 {}] zookeeper.ZKUtil(111): regionserver:45417-0x10143232da10001, quorum=127.0.0.1:64641, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b16ff38458c1,45417,1731752048912 2024-11-16T10:14:11,993 WARN [RS:0;b16ff38458c1:45417 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T10:14:11,993 INFO [RS:0;b16ff38458c1:45417 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T10:14:11,993 DEBUG [RS:0;b16ff38458c1:45417 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/WALs/b16ff38458c1,45417,1731752048912 2024-11-16T10:14:11,995 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b16ff38458c1,45417,1731752048912] 2024-11-16T10:14:12,021 INFO [RS:0;b16ff38458c1:45417 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T10:14:12,039 INFO [RS:0;b16ff38458c1:45417 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T10:14:12,044 INFO [RS:0;b16ff38458c1:45417 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T10:14:12,044 INFO [RS:0;b16ff38458c1:45417 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T10:14:12,045 INFO [RS:0;b16ff38458c1:45417 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T10:14:12,050 INFO [RS:0;b16ff38458c1:45417 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T10:14:12,051 INFO [RS:0;b16ff38458c1:45417 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T10:14:12,052 DEBUG [RS:0;b16ff38458c1:45417 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:14:12,052 DEBUG [RS:0;b16ff38458c1:45417 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:14:12,052 DEBUG [RS:0;b16ff38458c1:45417 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:14:12,052 DEBUG [RS:0;b16ff38458c1:45417 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:14:12,052 DEBUG [RS:0;b16ff38458c1:45417 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:14:12,052 DEBUG [RS:0;b16ff38458c1:45417 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b16ff38458c1:0, corePoolSize=2, maxPoolSize=2 2024-11-16T10:14:12,053 DEBUG [RS:0;b16ff38458c1:45417 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:14:12,053 DEBUG [RS:0;b16ff38458c1:45417 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:14:12,053 DEBUG [RS:0;b16ff38458c1:45417 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:14:12,053 DEBUG [RS:0;b16ff38458c1:45417 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:14:12,053 DEBUG [RS:0;b16ff38458c1:45417 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:14:12,053 DEBUG [RS:0;b16ff38458c1:45417 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:14:12,053 DEBUG [RS:0;b16ff38458c1:45417 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b16ff38458c1:0, corePoolSize=3, maxPoolSize=3 2024-11-16T10:14:12,054 DEBUG [RS:0;b16ff38458c1:45417 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0, corePoolSize=3, maxPoolSize=3 2024-11-16T10:14:12,054 INFO [RS:0;b16ff38458c1:45417 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T10:14:12,055 INFO [RS:0;b16ff38458c1:45417 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T10:14:12,055 INFO [RS:0;b16ff38458c1:45417 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T10:14:12,055 INFO [RS:0;b16ff38458c1:45417 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T10:14:12,055 INFO [RS:0;b16ff38458c1:45417 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T10:14:12,055 INFO [RS:0;b16ff38458c1:45417 {}] hbase.ChoreService(168): Chore ScheduledChore name=b16ff38458c1,45417,1731752048912-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T10:14:12,073 INFO [RS:0;b16ff38458c1:45417 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T10:14:12,075 INFO [RS:0;b16ff38458c1:45417 {}] hbase.ChoreService(168): Chore ScheduledChore name=b16ff38458c1,45417,1731752048912-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T10:14:12,075 INFO [RS:0;b16ff38458c1:45417 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T10:14:12,075 INFO [RS:0;b16ff38458c1:45417 {}] regionserver.Replication(171): b16ff38458c1,45417,1731752048912 started 2024-11-16T10:14:12,093 INFO [RS:0;b16ff38458c1:45417 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T10:14:12,093 INFO [RS:0;b16ff38458c1:45417 {}] regionserver.HRegionServer(1482): Serving as b16ff38458c1,45417,1731752048912, RpcServer on b16ff38458c1/172.17.0.2:45417, sessionid=0x10143232da10001 2024-11-16T10:14:12,094 DEBUG [RS:0;b16ff38458c1:45417 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T10:14:12,094 DEBUG [RS:0;b16ff38458c1:45417 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b16ff38458c1,45417,1731752048912 2024-11-16T10:14:12,094 DEBUG [RS:0;b16ff38458c1:45417 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b16ff38458c1,45417,1731752048912' 2024-11-16T10:14:12,094 DEBUG [RS:0;b16ff38458c1:45417 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T10:14:12,095 DEBUG [RS:0;b16ff38458c1:45417 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T10:14:12,096 DEBUG [RS:0;b16ff38458c1:45417 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T10:14:12,096 DEBUG [RS:0;b16ff38458c1:45417 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T10:14:12,096 DEBUG [RS:0;b16ff38458c1:45417 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b16ff38458c1,45417,1731752048912 2024-11-16T10:14:12,096 DEBUG [RS:0;b16ff38458c1:45417 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b16ff38458c1,45417,1731752048912' 2024-11-16T10:14:12,096 DEBUG [RS:0;b16ff38458c1:45417 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T10:14:12,097 DEBUG [RS:0;b16ff38458c1:45417 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T10:14:12,098 DEBUG [RS:0;b16ff38458c1:45417 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T10:14:12,098 INFO [RS:0;b16ff38458c1:45417 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T10:14:12,098 INFO [RS:0;b16ff38458c1:45417 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T10:14:12,205 INFO [RS:0;b16ff38458c1:45417 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T10:14:12,210 INFO [RS:0;b16ff38458c1:45417 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b16ff38458c1%2C45417%2C1731752048912, suffix=, logDir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/WALs/b16ff38458c1,45417,1731752048912, archiveDir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/oldWALs, maxLogs=32 2024-11-16T10:14:12,225 DEBUG [RS:0;b16ff38458c1:45417 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/WALs/b16ff38458c1,45417,1731752048912/b16ff38458c1%2C45417%2C1731752048912.1731752052213, exclude list is [], retry=0 2024-11-16T10:14:12,229 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44191,DS-daab0977-6a66-4666-bc1e-a135402836ed,DISK] 2024-11-16T10:14:12,232 INFO [RS:0;b16ff38458c1:45417 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/WALs/b16ff38458c1,45417,1731752048912/b16ff38458c1%2C45417%2C1731752048912.1731752052213 2024-11-16T10:14:12,232 DEBUG [RS:0;b16ff38458c1:45417 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40949:40949)] 2024-11-16T10:14:12,332 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T10:14:12,333 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0 2024-11-16T10:14:12,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741833_1009 (size=32) 2024-11-16T10:14:12,745 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:14:12,747 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T10:14:12,750 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T10:14:12,750 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:12,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T10:14:12,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T10:14:12,754 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T10:14:12,754 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:12,755 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T10:14:12,755 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T10:14:12,757 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T10:14:12,757 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:12,758 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T10:14:12,758 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T10:14:12,761 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T10:14:12,761 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:12,762 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T10:14:12,762 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T10:14:12,763 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740 2024-11-16T10:14:12,764 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740 2024-11-16T10:14:12,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T10:14:12,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T10:14:12,768 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:14:12,771 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T10:14:12,775 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T10:14:12,776 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72426545, jitterRate=0.079239621758461}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:14:12,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731752052745Initializing all the Stores at 1731752052747 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731752052747Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731752052747Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752052747Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731752052747Cleaning up temporary data from old regions at 1731752052767 (+20 ms)Region opened successfully at 1731752052780 (+13 ms) 2024-11-16T10:14:12,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T10:14:12,780 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T10:14:12,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T10:14:12,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T10:14:12,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T10:14:12,782 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T10:14:12,782 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731752052780Disabling compacts and flushes for region at 1731752052780Disabling writes for close at 1731752052780Writing region close event to WAL at 1731752052781 (+1 ms)Closed at 1731752052781 2024-11-16T10:14:12,785 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T10:14:12,785 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T10:14:12,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T10:14:12,801 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T10:14:12,804 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T10:14:12,962 DEBUG [b16ff38458c1:45949 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-16T10:14:12,973 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:12,979 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b16ff38458c1,45417,1731752048912, state=OPENING 2024-11-16T10:14:13,073 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T10:14:13,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45417-0x10143232da10001, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:14:13,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:14:13,088 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T10:14:13,088 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T10:14:13,089 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T10:14:13,091 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b16ff38458c1,45417,1731752048912}] 2024-11-16T10:14:13,271 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T10:14:13,274 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44731, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T10:14:13,284 INFO [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T10:14:13,284 INFO [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T10:14:13,285 INFO [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T10:14:13,288 INFO [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b16ff38458c1%2C45417%2C1731752048912.meta, suffix=.meta, logDir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/WALs/b16ff38458c1,45417,1731752048912, archiveDir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/oldWALs, maxLogs=32 2024-11-16T10:14:13,302 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/WALs/b16ff38458c1,45417,1731752048912/b16ff38458c1%2C45417%2C1731752048912.meta.1731752053290.meta, exclude list is [], retry=0 2024-11-16T10:14:13,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44191,DS-daab0977-6a66-4666-bc1e-a135402836ed,DISK] 2024-11-16T10:14:13,309 INFO [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/WALs/b16ff38458c1,45417,1731752048912/b16ff38458c1%2C45417%2C1731752048912.meta.1731752053290.meta 2024-11-16T10:14:13,309 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40949:40949)] 2024-11-16T10:14:13,309 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:14:13,311 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T10:14:13,314 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T10:14:13,318 INFO [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T10:14:13,322 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T10:14:13,323 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:14:13,323 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T10:14:13,323 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T10:14:13,326 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T10:14:13,328 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T10:14:13,328 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:13,329 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T10:14:13,329 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T10:14:13,330 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T10:14:13,330 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:13,331 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T10:14:13,331 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T10:14:13,333 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T10:14:13,333 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:13,334 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T10:14:13,334 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T10:14:13,335 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T10:14:13,335 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:13,336 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T10:14:13,336 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T10:14:13,338 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740 2024-11-16T10:14:13,341 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740 2024-11-16T10:14:13,344 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T10:14:13,344 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T10:14:13,345 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:14:13,348 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T10:14:13,350 INFO [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67984110, jitterRate=0.013042181730270386}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:14:13,350 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T10:14:13,352 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731752053324Writing region info on filesystem at 1731752053324Initializing all the Stores at 1731752053326 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731752053326Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731752053326Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752053326Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731752053326Cleaning up temporary data from old regions at 1731752053344 (+18 ms)Running coprocessor post-open hooks at 1731752053350 (+6 ms)Region opened successfully at 1731752053352 (+2 ms) 2024-11-16T10:14:13,358 INFO [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731752053261 2024-11-16T10:14:13,368 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T10:14:13,368 INFO [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T10:14:13,370 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:13,372 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b16ff38458c1,45417,1731752048912, state=OPEN 2024-11-16T10:14:13,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45417-0x10143232da10001, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T10:14:13,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T10:14:13,412 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T10:14:13,412 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T10:14:13,412 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:13,417 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T10:14:13,417 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b16ff38458c1,45417,1731752048912 in 321 msec 2024-11-16T10:14:13,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T10:14:13,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 627 msec 2024-11-16T10:14:13,426 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T10:14:13,426 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T10:14:13,444 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:14:13,445 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:14:13,462 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:14:13,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40163, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:14:13,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6490 sec 2024-11-16T10:14:13,483 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731752053483, completionTime=-1 2024-11-16T10:14:13,485 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-16T10:14:13,485 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-16T10:14:13,508 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-16T10:14:13,508 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731752113508 2024-11-16T10:14:13,509 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731752173509 2024-11-16T10:14:13,509 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 23 msec 2024-11-16T10:14:13,511 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b16ff38458c1,45949,1731752047888-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T10:14:13,511 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b16ff38458c1,45949,1731752047888-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T10:14:13,511 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b16ff38458c1,45949,1731752047888-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T10:14:13,512 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b16ff38458c1:45949, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T10:14:13,512 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T10:14:13,513 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T10:14:13,517 DEBUG [master/b16ff38458c1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T10:14:13,556 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.317sec 2024-11-16T10:14:13,558 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T10:14:13,559 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T10:14:13,560 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T10:14:13,561 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T10:14:13,561 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T10:14:13,562 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b16ff38458c1,45949,1731752047888-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T10:14:13,562 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b16ff38458c1,45949,1731752047888-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T10:14:13,570 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T10:14:13,574 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T10:14:13,574 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b16ff38458c1,45949,1731752047888-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T10:14:13,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:13,669 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-16T10:14:13,669 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-16T10:14:13,671 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:14:13,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:14:13,679 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:14:13,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:14:13,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:14:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a872003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:14:13,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:14:13,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:13,688 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:14:13,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:13,691 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:14:13,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:14:13,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:14:13,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:14:13,717 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b16ff38458c1,45949,1731752047888 2024-11-16T10:14:13,727 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=281, ProcessCount=11, AvailableMemoryMB=4499 2024-11-16T10:14:13,735 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T10:14:13,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is b16ff38458c1,45949,1731752047888 2024-11-16T10:14:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@56ef0928 2024-11-16T10:14:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T10:14:13,744 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T10:14:13,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T10:14:13,750 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T10:14:13,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T10:14:13,760 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T10:14:13,762 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-16T10:14:13,762 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:13,765 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T10:14:13,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T10:14:13,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741835_1011 (size=1061) 2024-11-16T10:14:13,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T10:14:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T10:14:14,184 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0 2024-11-16T10:14:14,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741836_1012 (size=53) 2024-11-16T10:14:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T10:14:14,597 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:14:14,598 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2bab7dc9c814fe637b384a46040e48b5, disabling compactions & flushes 2024-11-16T10:14:14,598 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:14,598 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:14,598 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. after waiting 0 ms 2024-11-16T10:14:14,598 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:14,599 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:14,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2bab7dc9c814fe637b384a46040e48b5: Waiting for close lock at 1731752054597Disabling compacts and flushes for region at 1731752054597Disabling writes for close at 1731752054598 (+1 ms)Writing region close event to WAL at 1731752054599 (+1 ms)Closed at 1731752054599 2024-11-16T10:14:14,603 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T10:14:14,612 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731752054603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731752054603"}]},"ts":"1731752054603"} 2024-11-16T10:14:14,617 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T10:14:14,619 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T10:14:14,621 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752054619"}]},"ts":"1731752054619"} 2024-11-16T10:14:14,625 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T10:14:14,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bab7dc9c814fe637b384a46040e48b5, ASSIGN}] 2024-11-16T10:14:14,630 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bab7dc9c814fe637b384a46040e48b5, ASSIGN 2024-11-16T10:14:14,631 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bab7dc9c814fe637b384a46040e48b5, ASSIGN; state=OFFLINE, location=b16ff38458c1,45417,1731752048912; forceNewPlan=false, retain=false 2024-11-16T10:14:14,782 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2bab7dc9c814fe637b384a46040e48b5, regionState=OPENING, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:14,787 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bab7dc9c814fe637b384a46040e48b5, ASSIGN because future has completed 2024-11-16T10:14:14,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912}] 2024-11-16T10:14:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T10:14:14,954 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:14,954 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:14:14,955 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:14,955 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:14:14,956 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:14,956 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:14,958 INFO [StoreOpener-2bab7dc9c814fe637b384a46040e48b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:14,962 INFO [StoreOpener-2bab7dc9c814fe637b384a46040e48b5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:14:14,963 INFO [StoreOpener-2bab7dc9c814fe637b384a46040e48b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bab7dc9c814fe637b384a46040e48b5 columnFamilyName A 2024-11-16T10:14:14,963 DEBUG [StoreOpener-2bab7dc9c814fe637b384a46040e48b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:14,964 INFO [StoreOpener-2bab7dc9c814fe637b384a46040e48b5-1 {}] regionserver.HStore(327): Store=2bab7dc9c814fe637b384a46040e48b5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:14:14,964 INFO [StoreOpener-2bab7dc9c814fe637b384a46040e48b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:14,966 INFO [StoreOpener-2bab7dc9c814fe637b384a46040e48b5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:14:14,967 INFO [StoreOpener-2bab7dc9c814fe637b384a46040e48b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bab7dc9c814fe637b384a46040e48b5 columnFamilyName B 2024-11-16T10:14:14,967 DEBUG [StoreOpener-2bab7dc9c814fe637b384a46040e48b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:14,968 INFO [StoreOpener-2bab7dc9c814fe637b384a46040e48b5-1 {}] regionserver.HStore(327): Store=2bab7dc9c814fe637b384a46040e48b5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:14:14,968 INFO [StoreOpener-2bab7dc9c814fe637b384a46040e48b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:14,970 INFO [StoreOpener-2bab7dc9c814fe637b384a46040e48b5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:14:14,971 INFO [StoreOpener-2bab7dc9c814fe637b384a46040e48b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bab7dc9c814fe637b384a46040e48b5 columnFamilyName C 2024-11-16T10:14:14,971 DEBUG [StoreOpener-2bab7dc9c814fe637b384a46040e48b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:14,972 INFO [StoreOpener-2bab7dc9c814fe637b384a46040e48b5-1 {}] regionserver.HStore(327): Store=2bab7dc9c814fe637b384a46040e48b5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:14:14,973 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:14,973 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:14,974 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:14,975 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:14,977 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:14,977 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:14,978 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:14:14,980 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:14,983 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T10:14:14,984 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 2bab7dc9c814fe637b384a46040e48b5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63145806, jitterRate=-0.059054166078567505}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:14:14,984 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:14,985 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 2bab7dc9c814fe637b384a46040e48b5: Running coprocessor pre-open hook at 1731752054956Writing region info on filesystem at 1731752054956Initializing all the Stores at 1731752054958 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752054958Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752054958Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752054958Cleaning up temporary data from old regions at 1731752054977 (+19 ms)Running coprocessor post-open hooks at 1731752054984 (+7 ms)Region opened successfully at 1731752054985 (+1 ms) 2024-11-16T10:14:14,987 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., pid=6, masterSystemTime=1731752054943 2024-11-16T10:14:14,990 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:14,990 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:14,991 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2bab7dc9c814fe637b384a46040e48b5, regionState=OPEN, openSeqNum=2, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:14,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 because future has completed 2024-11-16T10:14:15,000 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-16T10:14:15,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 in 208 msec 2024-11-16T10:14:15,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-16T10:14:15,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bab7dc9c814fe637b384a46040e48b5, ASSIGN in 373 msec 2024-11-16T10:14:15,007 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T10:14:15,007 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752055007"}]},"ts":"1731752055007"} 2024-11-16T10:14:15,010 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T10:14:15,011 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T10:14:15,015 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2580 sec 2024-11-16T10:14:15,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T10:14:15,928 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:14:15,935 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:15,935 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:14:15,935 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:14:15,937 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:14:15,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:14:15,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:14:15,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70cdc6cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:15,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:14:15,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:14:15,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:15,939 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:14:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1806c24b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:15,942 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:14:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:14:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:14:15,946 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:14:15,948 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T10:14:15,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is b16ff38458c1,45949,1731752047888 2024-11-16T10:14:15,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@78210a9b 2024-11-16T10:14:15,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T10:14:15,953 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T10:14:15,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:15,957 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:14:15,957 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:14:15,959 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:14:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:14:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:14:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149abd79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:14:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:14:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:15,962 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:14:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:15,964 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:15,964 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:14:15,965 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:14:15,966 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:14:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:14:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:14:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:14:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:14:15,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:15,969 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:14:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:15,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:15,972 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:14:15,972 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:14:15,974 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:14:15,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:14:15,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:14:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596d06e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:14:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:14:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:15,977 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:14:15,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:15,979 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:15,979 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:14:15,979 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:14:15,980 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:14:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:14:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:14:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa64271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:14:15,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:14:15,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:15,984 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:14:15,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76e341cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:15,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:15,986 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:14:15,986 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:14:15,987 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:14:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:14:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:14:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f695ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:14:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:14:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:15,989 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:14:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ebbd89f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:15,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:15,994 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:14:15,994 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:14:15,995 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:14:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:14:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:14:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3c1f14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:14:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:14:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:15,998 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:14:15,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e1b379, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:16,000 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:16,000 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:14:16,000 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:14:16,001 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:14:16,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:14:16,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:14:16,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20584f85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:16,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:14:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:14:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:16,003 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:14:16,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7aa9af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:16,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:16,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:14:16,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:14:16,006 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:14:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:14:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:14:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c287047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:14:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:14:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:16,009 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:14:16,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52e2fecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:16,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:16,010 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:14:16,011 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:14:16,012 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:14:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:14:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:14:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@311f9e8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:14:16,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:14:16,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:16,014 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:14:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f696c27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:16,019 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:14:16,019 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:14:16,019 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:14:16,019 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:14:16,019 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:14:16,019 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:14:16,020 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:14:16,021 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:14:16,021 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:14:16,022 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:14:16,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:14:16,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:14:16,022 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:14:16,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:14:16,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:14:16,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:14:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:14:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:14:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:14:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:14:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:14:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:14:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:14:16,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:14:16,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:14:16,025 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:14:16,025 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:14:16,025 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:14:16,025 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:14:16,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:14:16,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:14:16,026 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:14:16,027 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:14:16,027 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:14:16,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:14:16,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:14:16,031 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:14:16,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-16T10:14:16,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:14:16,036 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:14:16,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:14:16,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:14:16,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:14:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:14:16,042 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:14:16,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:14:16,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:14:16,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:14:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T10:14:16,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:14:16,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:14:16,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:16,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T10:14:16,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:16,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:16,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:16,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:16,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:16,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:16,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T10:14:16,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T10:14:16,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:16,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:16,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:16,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:16,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:16,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:16,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:16,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752056201, max timestamp=1731752056222Num uniques 859; 2024-11-16T10:14:16,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731752056201, max timestamp=1731752056222Num uniques 859; 2024-11-16T10:14:16,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:14:16,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T10:14:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752116214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:16,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752116212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:16,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/4a6ee23f28ab4e28b1732549b25bf823 is 50, key is test_row_0/A:col10/1731752056092/Put/seqid=0 2024-11-16T10:14:16,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752056201, max timestamp=1731752056222Num uniques 859; 2024-11-16T10:14:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:16,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:14:16,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:16,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741837_1013 (size=12001) 2024-11-16T10:14:16,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752116347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:16,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:16,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752116351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:16,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752116351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:16,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:16,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:16,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T10:14:16,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752116385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:16,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:16,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752116385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:16,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T10:14:16,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:16,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:16,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:16,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:16,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:16,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752116465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:16,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752116466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:16,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:16,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752116477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:16,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:16,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T10:14:16,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:16,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:16,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:16,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:16,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:16,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752116594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:16,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:16,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:16,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752116605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:16,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:16,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T10:14:16,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/4a6ee23f28ab4e28b1732549b25bf823 2024-11-16T10:14:16,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:16,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752116684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:16,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:16,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:16,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752116685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:16,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:16,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:16,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752116694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:16,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:16,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T10:14:16,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:16,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:16,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:16,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:16,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:16,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/1913d37f8ca640e9a7b1ae6b69b3bc92 is 50, key is test_row_0/B:col10/1731752056092/Put/seqid=0 2024-11-16T10:14:16,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741838_1014 (size=12001) 2024-11-16T10:14:16,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T10:14:16,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:16,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:16,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:16,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:16,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:16,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752116915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:16,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752116915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:16,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:16,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752116994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:16,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:16,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:17,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752117004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:17,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752117004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:17,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T10:14:17,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:17,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:17,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:17,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:17,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T10:14:17,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T10:14:17,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:17,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:17,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:17,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:17,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:17,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/1913d37f8ca640e9a7b1ae6b69b3bc92 2024-11-16T10:14:17,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:17,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/04f6e1d9288944e0a5b94e1239655795 is 50, key is test_row_0/C:col10/1731752056092/Put/seqid=0 2024-11-16T10:14:17,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741839_1015 (size=12001) 2024-11-16T10:14:17,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T10:14:17,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:17,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:17,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:17,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:17,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:17,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:17,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:17,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752117425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:17,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:17,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:17,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:17,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:17,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752117425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:17,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:17,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752117505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:17,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:17,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:17,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:17,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:17,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752117514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:17,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:17,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:17,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:17,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T10:14:17,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:17,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:17,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:17,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:17,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:17,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:17,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:17,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752117525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:17,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:17,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:17,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:17,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/04f6e1d9288944e0a5b94e1239655795 2024-11-16T10:14:17,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T10:14:17,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:17,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:17,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:17,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:17,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:17,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/4a6ee23f28ab4e28b1732549b25bf823 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/4a6ee23f28ab4e28b1732549b25bf823 2024-11-16T10:14:17,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/4a6ee23f28ab4e28b1732549b25bf823, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T10:14:17,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/1913d37f8ca640e9a7b1ae6b69b3bc92 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/1913d37f8ca640e9a7b1ae6b69b3bc92 2024-11-16T10:14:17,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/1913d37f8ca640e9a7b1ae6b69b3bc92, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T10:14:17,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/04f6e1d9288944e0a5b94e1239655795 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/04f6e1d9288944e0a5b94e1239655795 2024-11-16T10:14:17,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/04f6e1d9288944e0a5b94e1239655795, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T10:14:17,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=123.00 KB/125950 for 2bab7dc9c814fe637b384a46040e48b5 in 1655ms, sequenceid=13, compaction requested=false 2024-11-16T10:14:17,769 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-16T10:14:17,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:17,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T10:14:17,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:17,840 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=123.00 KB heapSize=345.53 KB 2024-11-16T10:14:17,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:17,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:17,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:17,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:17,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:17,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:17,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/03c54c9fc3f94b0eb16a177c60a81d6d is 50, key is test_row_0/A:col10/1731752056326/Put/seqid=0 2024-11-16T10:14:17,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741840_1016 (size=12001) 2024-11-16T10:14:17,880 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/03c54c9fc3f94b0eb16a177c60a81d6d 2024-11-16T10:14:17,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/1e835da9a95d4eeab64270f9d41e6cf6 is 50, key is test_row_0/B:col10/1731752056326/Put/seqid=0 2024-11-16T10:14:17,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741841_1017 (size=12001) 2024-11-16T10:14:17,931 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/1e835da9a95d4eeab64270f9d41e6cf6 2024-11-16T10:14:17,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/8576be72d18544f5930f9edd974a5021 is 50, key is test_row_0/C:col10/1731752056326/Put/seqid=0 2024-11-16T10:14:17,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741842_1018 (size=12001) 2024-11-16T10:14:17,976 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/8576be72d18544f5930f9edd974a5021 2024-11-16T10:14:17,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/03c54c9fc3f94b0eb16a177c60a81d6d as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/03c54c9fc3f94b0eb16a177c60a81d6d 2024-11-16T10:14:18,014 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-16T10:14:18,018 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/03c54c9fc3f94b0eb16a177c60a81d6d, entries=150, sequenceid=47, filesize=11.7 K 2024-11-16T10:14:18,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/1e835da9a95d4eeab64270f9d41e6cf6 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/1e835da9a95d4eeab64270f9d41e6cf6 2024-11-16T10:14:18,036 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/1e835da9a95d4eeab64270f9d41e6cf6, entries=150, sequenceid=47, filesize=11.7 K 2024-11-16T10:14:18,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/8576be72d18544f5930f9edd974a5021 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/8576be72d18544f5930f9edd974a5021 2024-11-16T10:14:18,055 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/8576be72d18544f5930f9edd974a5021, entries=150, sequenceid=47, filesize=11.7 K 2024-11-16T10:14:18,060 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~123.00 KB/125950, heapSize ~345.48 KB/353776, currentSize=0 B/0 for 2bab7dc9c814fe637b384a46040e48b5 in 220ms, sequenceid=47, compaction requested=false 2024-11-16T10:14:18,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:18,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:18,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-16T10:14:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-16T10:14:18,071 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-16T10:14:18,071 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0190 sec 2024-11-16T10:14:18,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.0450 sec 2024-11-16T10:14:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T10:14:18,195 INFO [Async-Client-Retry-Timer-pool-0 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:14:18,196 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:14:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-16T10:14:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T10:14:18,206 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:14:18,208 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:14:18,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:14:18,249 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-16T10:14:18,250 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-16T10:14:18,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T10:14:18,261 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-16T10:14:18,261 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-16T10:14:18,261 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-16T10:14:18,264 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T10:14:18,264 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-16T10:14:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T10:14:18,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T10:14:18,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:18,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:18,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:18,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-16T10:14:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-16T10:14:18,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-16T10:14:18,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 161 msec 2024-11-16T10:14:18,380 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 181 msec 2024-11-16T10:14:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T10:14:18,525 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:14:18,526 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:14:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-16T10:14:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T10:14:18,530 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:14:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:18,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T10:14:18,532 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:14:18,532 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:14:18,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:18,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:18,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:18,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:18,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:18,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:18,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/fe096d1e5553454298c10a7c04eb5f41 is 50, key is test_row_0/A:col10/1731752058529/Put/seqid=0 2024-11-16T10:14:18,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741843_1019 (size=11997) 2024-11-16T10:14:18,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/fe096d1e5553454298c10a7c04eb5f41 2024-11-16T10:14:18,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752058570, max timestamp=1731752058592Num uniques 859; 2024-11-16T10:14:18,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:14:18,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752058570, max timestamp=1731752058593Num uniques 859; 2024-11-16T10:14:18,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:14:18,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752058570, max timestamp=1731752058592Num uniques 859; 2024-11-16T10:14:18,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:14:18,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/a9c5966d6e524b088aefcbe441ceeafb is 50, key is test_row_0/B:col10/1731752058529/Put/seqid=0 2024-11-16T10:14:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T10:14:18,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741844_1020 (size=9657) 2024-11-16T10:14:18,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/a9c5966d6e524b088aefcbe441ceeafb 2024-11-16T10:14:18,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/4bc59a12e99d46f4bf3678b4d5956c6b is 50, key is test_row_0/C:col10/1731752058529/Put/seqid=0 2024-11-16T10:14:18,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741845_1021 (size=9657) 2024-11-16T10:14:18,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T10:14:18,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:18,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:18,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:18,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:18,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:18,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752118688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:18,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752118689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:18,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752118690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:18,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:18,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752118693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:18,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752118694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:18,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752118804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:18,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:18,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:18,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:18,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752118815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:18,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752118815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:18,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752118815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:18,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752118815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T10:14:18,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T10:14:18,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:18,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:18,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:18,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:18,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:18,849 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T10:14:19,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T10:14:19,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:19,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:19,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:19,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:19,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:19,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752119014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:19,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:19,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752119024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:19,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:19,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:19,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752119025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752119025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:19,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752119025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:19,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:19,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/4bc59a12e99d46f4bf3678b4d5956c6b 2024-11-16T10:14:19,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/fe096d1e5553454298c10a7c04eb5f41 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/fe096d1e5553454298c10a7c04eb5f41 2024-11-16T10:14:19,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/fe096d1e5553454298c10a7c04eb5f41, entries=150, sequenceid=59, filesize=11.7 K 2024-11-16T10:14:19,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/a9c5966d6e524b088aefcbe441ceeafb as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a9c5966d6e524b088aefcbe441ceeafb 2024-11-16T10:14:19,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a9c5966d6e524b088aefcbe441ceeafb, entries=100, sequenceid=59, filesize=9.4 K 2024-11-16T10:14:19,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/4bc59a12e99d46f4bf3678b4d5956c6b as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/4bc59a12e99d46f4bf3678b4d5956c6b 2024-11-16T10:14:19,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/4bc59a12e99d46f4bf3678b4d5956c6b, entries=100, sequenceid=59, filesize=9.4 K 2024-11-16T10:14:19,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 2bab7dc9c814fe637b384a46040e48b5 in 616ms, sequenceid=59, compaction requested=true 2024-11-16T10:14:19,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:19,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:14:19,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:19,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:14:19,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:19,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:14:19,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:19,154 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T10:14:19,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T10:14:19,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:19,159 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:19,159 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-16T10:14:19,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:19,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:19,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:19,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:19,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:19,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:19,163 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:19,165 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/B is initiating minor compaction (all files) 2024-11-16T10:14:19,166 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/B in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:19,167 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:19,167 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/A is initiating minor compaction (all files) 2024-11-16T10:14:19,167 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/A in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:19,168 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/1913d37f8ca640e9a7b1ae6b69b3bc92, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/1e835da9a95d4eeab64270f9d41e6cf6, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a9c5966d6e524b088aefcbe441ceeafb] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=32.9 K 2024-11-16T10:14:19,172 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 1913d37f8ca640e9a7b1ae6b69b3bc92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731752056092 2024-11-16T10:14:19,172 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/4a6ee23f28ab4e28b1732549b25bf823, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/03c54c9fc3f94b0eb16a177c60a81d6d, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/fe096d1e5553454298c10a7c04eb5f41] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=35.2 K 2024-11-16T10:14:19,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/0dff2173001444cfb847c2c7f6f90e2e is 50, key is test_row_0/A:col10/1731752058687/Put/seqid=0 2024-11-16T10:14:19,177 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e835da9a95d4eeab64270f9d41e6cf6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731752056326 2024-11-16T10:14:19,178 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting a9c5966d6e524b088aefcbe441ceeafb, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731752058519 2024-11-16T10:14:19,180 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a6ee23f28ab4e28b1732549b25bf823, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731752056092 2024-11-16T10:14:19,182 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03c54c9fc3f94b0eb16a177c60a81d6d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731752056326 2024-11-16T10:14:19,186 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe096d1e5553454298c10a7c04eb5f41, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731752058519 2024-11-16T10:14:19,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741846_1022 (size=12001) 2024-11-16T10:14:19,241 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#A#compaction#10 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:19,242 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#B#compaction#11 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:19,242 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/1a0dc691bc8f41efb7ab04a2e509a60c is 50, key is test_row_0/A:col10/1731752058529/Put/seqid=0 2024-11-16T10:14:19,251 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/09e1e8bce8e748c6b5db0a89e26233ba is 50, key is test_row_0/B:col10/1731752058529/Put/seqid=0 2024-11-16T10:14:19,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741847_1023 (size=12104) 2024-11-16T10:14:19,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741848_1024 (size=12104) 2024-11-16T10:14:19,307 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/1a0dc691bc8f41efb7ab04a2e509a60c as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/1a0dc691bc8f41efb7ab04a2e509a60c 2024-11-16T10:14:19,312 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/09e1e8bce8e748c6b5db0a89e26233ba as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/09e1e8bce8e748c6b5db0a89e26233ba 2024-11-16T10:14:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:19,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:19,336 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/A of 2bab7dc9c814fe637b384a46040e48b5 into 1a0dc691bc8f41efb7ab04a2e509a60c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:19,336 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/B of 2bab7dc9c814fe637b384a46040e48b5 into 09e1e8bce8e748c6b5db0a89e26233ba(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:19,336 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:19,336 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:19,339 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/A, priority=13, startTime=1731752059148; duration=0sec 2024-11-16T10:14:19,339 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/B, priority=13, startTime=1731752059151; duration=0sec 2024-11-16T10:14:19,340 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:19,340 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:19,340 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:B 2024-11-16T10:14:19,340 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:A 2024-11-16T10:14:19,340 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:19,343 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:19,343 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/C is initiating minor compaction (all files) 2024-11-16T10:14:19,343 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/C in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:19,344 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/04f6e1d9288944e0a5b94e1239655795, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/8576be72d18544f5930f9edd974a5021, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/4bc59a12e99d46f4bf3678b4d5956c6b] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=32.9 K 2024-11-16T10:14:19,347 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04f6e1d9288944e0a5b94e1239655795, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731752056092 2024-11-16T10:14:19,348 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8576be72d18544f5930f9edd974a5021, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731752056326 2024-11-16T10:14:19,349 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bc59a12e99d46f4bf3678b4d5956c6b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731752058519 2024-11-16T10:14:19,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752119355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:19,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:19,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752119355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:19,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752119359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:19,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:19,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752119363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:19,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:19,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:19,376 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#C#compaction#12 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:19,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752119364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:19,377 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/84559caf512843578976691f461ffa0a is 50, key is test_row_0/C:col10/1731752058529/Put/seqid=0 2024-11-16T10:14:19,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:19,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741849_1025 (size=12104) 2024-11-16T10:14:19,415 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/84559caf512843578976691f461ffa0a as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/84559caf512843578976691f461ffa0a 2024-11-16T10:14:19,441 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/C of 2bab7dc9c814fe637b384a46040e48b5 into 84559caf512843578976691f461ffa0a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:19,441 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:19,441 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/C, priority=13, startTime=1731752059152; duration=0sec 2024-11-16T10:14:19,441 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:19,442 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:C 2024-11-16T10:14:19,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752119475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752119475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:19,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752119475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:19,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:19,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:19,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:19,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752119484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752119484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:19,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:19,595 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/0dff2173001444cfb847c2c7f6f90e2e 2024-11-16T10:14:19,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/13917a7b449648bbb94ac3e3fd509289 is 50, key is test_row_0/B:col10/1731752058687/Put/seqid=0 2024-11-16T10:14:19,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741850_1026 (size=12001) 2024-11-16T10:14:19,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T10:14:19,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:19,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:19,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752119684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:19,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752119685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:19,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:19,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:19,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752119685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:19,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:19,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:19,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752119704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:19,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:19,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:19,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752119704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:19,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:19,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:19,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:19,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:19,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752119994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:20,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752119994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:20,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:20,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:20,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:20,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752120004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:20,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:20,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752120014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:20,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:20,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:20,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:20,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:20,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752120024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:20,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:20,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:20,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:20,043 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/13917a7b449648bbb94ac3e3fd509289 2024-11-16T10:14:20,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/2897ef71927e4086b771847973b9b33f is 50, key is test_row_0/C:col10/1731752058687/Put/seqid=0 2024-11-16T10:14:20,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741851_1027 (size=12001) 2024-11-16T10:14:20,485 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/2897ef71927e4086b771847973b9b33f 2024-11-16T10:14:20,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/0dff2173001444cfb847c2c7f6f90e2e as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/0dff2173001444cfb847c2c7f6f90e2e 2024-11-16T10:14:20,516 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/0dff2173001444cfb847c2c7f6f90e2e, entries=150, sequenceid=95, filesize=11.7 K 2024-11-16T10:14:20,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752120514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:20,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/13917a7b449648bbb94ac3e3fd509289 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/13917a7b449648bbb94ac3e3fd509289 2024-11-16T10:14:20,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:20,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:20,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:20,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752120514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:20,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:20,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:20,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:20,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752120515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:20,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:20,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:20,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:20,529 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/13917a7b449648bbb94ac3e3fd509289, entries=150, sequenceid=95, filesize=11.7 K 2024-11-16T10:14:20,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/2897ef71927e4086b771847973b9b33f as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/2897ef71927e4086b771847973b9b33f 2024-11-16T10:14:20,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:20,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752120534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:20,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:20,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:20,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:20,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752120534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:20,549 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/2897ef71927e4086b771847973b9b33f, entries=150, sequenceid=95, filesize=11.7 K 2024-11-16T10:14:20,551 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 2bab7dc9c814fe637b384a46040e48b5 in 1391ms, sequenceid=95, compaction requested=false 2024-11-16T10:14:20,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:20,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:20,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-16T10:14:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-16T10:14:20,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-16T10:14:20,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0210 sec 2024-11-16T10:14:20,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.0330 sec 2024-11-16T10:14:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T10:14:20,685 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:14:20,686 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:14:20,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-16T10:14:20,689 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:14:20,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T10:14:20,694 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:14:20,694 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:14:20,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T10:14:20,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T10:14:20,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:20,850 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T10:14:20,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:20,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:20,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:20,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:20,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:20,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:20,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/d49c3bb5f9674e36a000a56dd47f6ccd is 50, key is test_row_0/A:col10/1731752059342/Put/seqid=0 2024-11-16T10:14:20,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741852_1028 (size=12001) 2024-11-16T10:14:20,875 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/d49c3bb5f9674e36a000a56dd47f6ccd 2024-11-16T10:14:20,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/a0ffbc669a594e7da3b5104a3e3eaebe is 50, key is test_row_0/B:col10/1731752059342/Put/seqid=0 2024-11-16T10:14:20,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741853_1029 (size=12001) 2024-11-16T10:14:20,907 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/a0ffbc669a594e7da3b5104a3e3eaebe 2024-11-16T10:14:20,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/2e46bd55c2cd48faa4e9b178527dfa82 is 50, key is test_row_0/C:col10/1731752059342/Put/seqid=0 2024-11-16T10:14:20,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741854_1030 (size=12001) 2024-11-16T10:14:20,952 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/2e46bd55c2cd48faa4e9b178527dfa82 2024-11-16T10:14:20,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/d49c3bb5f9674e36a000a56dd47f6ccd as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/d49c3bb5f9674e36a000a56dd47f6ccd 2024-11-16T10:14:20,982 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/d49c3bb5f9674e36a000a56dd47f6ccd, entries=150, sequenceid=109, filesize=11.7 K 2024-11-16T10:14:20,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/a0ffbc669a594e7da3b5104a3e3eaebe as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a0ffbc669a594e7da3b5104a3e3eaebe 2024-11-16T10:14:20,997 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a0ffbc669a594e7da3b5104a3e3eaebe, entries=150, sequenceid=109, filesize=11.7 K 2024-11-16T10:14:21,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/2e46bd55c2cd48faa4e9b178527dfa82 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/2e46bd55c2cd48faa4e9b178527dfa82 2024-11-16T10:14:21,010 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/2e46bd55c2cd48faa4e9b178527dfa82, entries=150, sequenceid=109, filesize=11.7 K 2024-11-16T10:14:21,012 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for 2bab7dc9c814fe637b384a46040e48b5 in 162ms, sequenceid=109, compaction requested=true 2024-11-16T10:14:21,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:21,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:21,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-16T10:14:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-16T10:14:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T10:14:21,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-16T10:14:21,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 321 msec 2024-11-16T10:14:21,023 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 335 msec 2024-11-16T10:14:21,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T10:14:21,324 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:14:21,325 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:14:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-16T10:14:21,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T10:14:21,329 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:14:21,331 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:14:21,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:14:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T10:14:21,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T10:14:21,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:21,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:21,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:21,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-16T10:14:21,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-16T10:14:21,520 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-16T10:14:21,520 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 164 msec 2024-11-16T10:14:21,524 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 197 msec 2024-11-16T10:14:21,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T10:14:21,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:21,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:21,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:21,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:21,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:21,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:21,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:21,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/40c8cb5845a04993b6c9d14eb8e8c207 is 50, key is test_row_0/A:col10/1731752061542/Put/seqid=0 2024-11-16T10:14:21,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741855_1031 (size=12001) 2024-11-16T10:14:21,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/40c8cb5845a04993b6c9d14eb8e8c207 2024-11-16T10:14:21,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/6c86d9862fc84da99d664698cfa75a02 is 50, key is test_row_0/B:col10/1731752061542/Put/seqid=0 2024-11-16T10:14:21,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741856_1032 (size=12001) 2024-11-16T10:14:21,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752121592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752121594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752061577, max timestamp=1731752061601Num uniques 859; 2024-11-16T10:14:21,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752061577, max timestamp=1731752061601Num uniques 859; 2024-11-16T10:14:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:21,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:21,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:14:21,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:14:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752121597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:21,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752061577, max timestamp=1731752061603Num uniques 859; 2024-11-16T10:14:21,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:14:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T10:14:21,645 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:14:21,645 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:14:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-16T10:14:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T10:14:21,647 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:14:21,650 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:14:21,650 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:14:21,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752121661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:21,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752121662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:21,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752121715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:21,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752121715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:21,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752121715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:21,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T10:14:21,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752121775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:21,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752121784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:21,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:21,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T10:14:21,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:21,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:21,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:21,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:21,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:21,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752121934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:21,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752121935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752121935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:21,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T10:14:21,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:21,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:21,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:21,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:21,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T10:14:21,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752121985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:21,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:21,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:22,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/6c86d9862fc84da99d664698cfa75a02 2024-11-16T10:14:22,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752122004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:22,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/b88225d89ccf4691aac95f989247a132 is 50, key is test_row_0/C:col10/1731752061542/Put/seqid=0 2024-11-16T10:14:22,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741857_1033 (size=12001) 2024-11-16T10:14:22,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/b88225d89ccf4691aac95f989247a132 2024-11-16T10:14:22,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/40c8cb5845a04993b6c9d14eb8e8c207 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/40c8cb5845a04993b6c9d14eb8e8c207 2024-11-16T10:14:22,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/40c8cb5845a04993b6c9d14eb8e8c207, entries=150, sequenceid=121, filesize=11.7 K 2024-11-16T10:14:22,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/6c86d9862fc84da99d664698cfa75a02 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6c86d9862fc84da99d664698cfa75a02 2024-11-16T10:14:22,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6c86d9862fc84da99d664698cfa75a02, entries=150, sequenceid=121, filesize=11.7 K 2024-11-16T10:14:22,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/b88225d89ccf4691aac95f989247a132 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/b88225d89ccf4691aac95f989247a132 2024-11-16T10:14:22,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/b88225d89ccf4691aac95f989247a132, entries=150, sequenceid=121, filesize=11.7 K 2024-11-16T10:14:22,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 2bab7dc9c814fe637b384a46040e48b5 in 566ms, sequenceid=121, compaction requested=true 2024-11-16T10:14:22,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:22,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:14:22,110 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:14:22,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:22,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:14:22,111 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:14:22,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:22,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:14:22,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:22,113 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:14:22,113 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/A is initiating minor compaction (all files) 2024-11-16T10:14:22,113 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:14:22,113 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/A in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:22,113 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/B is initiating minor compaction (all files) 2024-11-16T10:14:22,113 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/B in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:22,113 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/1a0dc691bc8f41efb7ab04a2e509a60c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/0dff2173001444cfb847c2c7f6f90e2e, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/d49c3bb5f9674e36a000a56dd47f6ccd, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/40c8cb5845a04993b6c9d14eb8e8c207] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=47.0 K 2024-11-16T10:14:22,113 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/09e1e8bce8e748c6b5db0a89e26233ba, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/13917a7b449648bbb94ac3e3fd509289, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a0ffbc669a594e7da3b5104a3e3eaebe, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6c86d9862fc84da99d664698cfa75a02] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=47.0 K 2024-11-16T10:14:22,114 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a0dc691bc8f41efb7ab04a2e509a60c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731752056345 2024-11-16T10:14:22,114 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 09e1e8bce8e748c6b5db0a89e26233ba, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731752056345 2024-11-16T10:14:22,115 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dff2173001444cfb847c2c7f6f90e2e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731752058648 2024-11-16T10:14:22,115 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 13917a7b449648bbb94ac3e3fd509289, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731752058648 2024-11-16T10:14:22,115 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting d49c3bb5f9674e36a000a56dd47f6ccd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731752059342 2024-11-16T10:14:22,115 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting a0ffbc669a594e7da3b5104a3e3eaebe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731752059342 2024-11-16T10:14:22,116 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40c8cb5845a04993b6c9d14eb8e8c207, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731752061536 2024-11-16T10:14:22,116 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c86d9862fc84da99d664698cfa75a02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731752061536 2024-11-16T10:14:22,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T10:14:22,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:22,118 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-16T10:14:22,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:22,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:22,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:22,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:22,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:22,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:22,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/1115ad2bb2534544b8661c0169c6ff31 is 50, key is test_row_0/A:col10/1731752061659/Put/seqid=0 2024-11-16T10:14:22,152 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#A#compaction#22 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:22,153 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#B#compaction#23 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:22,153 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/dd8c6aa16a5e418e9cfbc6587d333ee1 is 50, key is test_row_0/A:col10/1731752061542/Put/seqid=0 2024-11-16T10:14:22,154 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/2663a08d1c454300a159faa8af6ceaf2 is 50, key is test_row_0/B:col10/1731752061542/Put/seqid=0 2024-11-16T10:14:22,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741858_1034 (size=12151) 2024-11-16T10:14:22,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741859_1035 (size=12241) 2024-11-16T10:14:22,186 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/1115ad2bb2534544b8661c0169c6ff31 2024-11-16T10:14:22,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741860_1036 (size=12241) 2024-11-16T10:14:22,200 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/2663a08d1c454300a159faa8af6ceaf2 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/2663a08d1c454300a159faa8af6ceaf2 2024-11-16T10:14:22,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/9b353d14d376437eb1e8fbf4ae64d39c is 50, key is test_row_0/B:col10/1731752061659/Put/seqid=0 2024-11-16T10:14:22,219 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/B of 2bab7dc9c814fe637b384a46040e48b5 into 2663a08d1c454300a159faa8af6ceaf2(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:22,219 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:22,219 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/B, priority=12, startTime=1731752062110; duration=0sec 2024-11-16T10:14:22,219 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:22,219 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:B 2024-11-16T10:14:22,220 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:14:22,221 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:14:22,222 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/C is initiating minor compaction (all files) 2024-11-16T10:14:22,222 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/C in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:22,222 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/84559caf512843578976691f461ffa0a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/2897ef71927e4086b771847973b9b33f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/2e46bd55c2cd48faa4e9b178527dfa82, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/b88225d89ccf4691aac95f989247a132] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=47.0 K 2024-11-16T10:14:22,224 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 84559caf512843578976691f461ffa0a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731752056345 2024-11-16T10:14:22,227 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 2897ef71927e4086b771847973b9b33f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731752058648 2024-11-16T10:14:22,229 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e46bd55c2cd48faa4e9b178527dfa82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731752059342 2024-11-16T10:14:22,230 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting b88225d89ccf4691aac95f989247a132, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731752061536 2024-11-16T10:14:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:22,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:22,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741861_1037 (size=12151) 2024-11-16T10:14:22,260 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/9b353d14d376437eb1e8fbf4ae64d39c 2024-11-16T10:14:22,262 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#C#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:22,263 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/cd1878fea611407fb24c0d373ebb2977 is 50, key is test_row_0/C:col10/1731752061542/Put/seqid=0 2024-11-16T10:14:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T10:14:22,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/6596d32419ea4269afbccbf8db8dd914 is 50, key is test_row_0/C:col10/1731752061659/Put/seqid=0 2024-11-16T10:14:22,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741862_1038 (size=12241) 2024-11-16T10:14:22,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752122285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:22,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:22,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752122295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:22,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741863_1039 (size=12151) 2024-11-16T10:14:22,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752122303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:22,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752122308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:22,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752122324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:22,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:22,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752122415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:22,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752122415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:22,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752122415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:22,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:22,607 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/dd8c6aa16a5e418e9cfbc6587d333ee1 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/dd8c6aa16a5e418e9cfbc6587d333ee1 2024-11-16T10:14:22,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752122624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:22,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752122625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:22,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:22,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752122634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:22,646 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/A of 2bab7dc9c814fe637b384a46040e48b5 into dd8c6aa16a5e418e9cfbc6587d333ee1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:22,646 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:22,646 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/A, priority=12, startTime=1731752062110; duration=0sec 2024-11-16T10:14:22,647 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:22,647 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:A 2024-11-16T10:14:22,704 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/6596d32419ea4269afbccbf8db8dd914 2024-11-16T10:14:22,705 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/cd1878fea611407fb24c0d373ebb2977 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/cd1878fea611407fb24c0d373ebb2977 2024-11-16T10:14:22,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/1115ad2bb2534544b8661c0169c6ff31 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/1115ad2bb2534544b8661c0169c6ff31 2024-11-16T10:14:22,716 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/C of 2bab7dc9c814fe637b384a46040e48b5 into cd1878fea611407fb24c0d373ebb2977(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:22,716 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:22,716 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/C, priority=12, startTime=1731752062111; duration=0sec 2024-11-16T10:14:22,716 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:22,716 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:C 2024-11-16T10:14:22,721 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/1115ad2bb2534544b8661c0169c6ff31, entries=150, sequenceid=157, filesize=11.9 K 2024-11-16T10:14:22,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/9b353d14d376437eb1e8fbf4ae64d39c as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/9b353d14d376437eb1e8fbf4ae64d39c 2024-11-16T10:14:22,733 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/9b353d14d376437eb1e8fbf4ae64d39c, entries=150, sequenceid=157, filesize=11.9 K 2024-11-16T10:14:22,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/6596d32419ea4269afbccbf8db8dd914 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/6596d32419ea4269afbccbf8db8dd914 2024-11-16T10:14:22,742 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/6596d32419ea4269afbccbf8db8dd914, entries=150, sequenceid=157, filesize=11.9 K 2024-11-16T10:14:22,744 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=60.38 KB/61830 for 2bab7dc9c814fe637b384a46040e48b5 in 626ms, sequenceid=157, compaction requested=false 2024-11-16T10:14:22,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:22,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:22,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-16T10:14:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-16T10:14:22,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-16T10:14:22,760 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0970 sec 2024-11-16T10:14:22,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.1160 sec 2024-11-16T10:14:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T10:14:22,785 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:14:22,786 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:14:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-16T10:14:22,797 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:14:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T10:14:22,801 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:14:22,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:14:22,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T10:14:22,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:22,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:22,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:22,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:22,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:22,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:22,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/c65f5bb7856d427596a33aa19a006ecc is 50, key is test_row_0/A:col10/1731752062300/Put/seqid=0 2024-11-16T10:14:22,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741864_1040 (size=14541) 2024-11-16T10:14:22,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/c65f5bb7856d427596a33aa19a006ecc 2024-11-16T10:14:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T10:14:22,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/c7576978ee234523a8abe3ef6e278c66 is 50, key is test_row_0/B:col10/1731752062300/Put/seqid=0 2024-11-16T10:14:22,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752122923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:22,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752122926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:22,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:22,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752122940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:22,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752122940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:22,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:22,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752122944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:22,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:22,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:22,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741865_1041 (size=12151) 2024-11-16T10:14:22,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/c7576978ee234523a8abe3ef6e278c66 2024-11-16T10:14:22,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T10:14:22,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:22,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:22,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:22,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:22,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:22,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/cf9aaa7a34ac4371822303d9480217e9 is 50, key is test_row_0/C:col10/1731752062300/Put/seqid=0 2024-11-16T10:14:23,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741866_1042 (size=12151) 2024-11-16T10:14:23,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752123044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:23,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752123054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:23,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T10:14:23,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T10:14:23,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:23,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:23,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:23,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:23,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:23,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752123254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:23,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752123264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:23,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:23,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T10:14:23,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:23,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:23,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:23,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:23,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:23,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/cf9aaa7a34ac4371822303d9480217e9 2024-11-16T10:14:23,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/c65f5bb7856d427596a33aa19a006ecc as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/c65f5bb7856d427596a33aa19a006ecc 2024-11-16T10:14:23,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/c65f5bb7856d427596a33aa19a006ecc, entries=200, sequenceid=173, filesize=14.2 K 2024-11-16T10:14:23,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/c7576978ee234523a8abe3ef6e278c66 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/c7576978ee234523a8abe3ef6e278c66 2024-11-16T10:14:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T10:14:23,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T10:14:23,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:23,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:23,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:23,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:23,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:23,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/c7576978ee234523a8abe3ef6e278c66, entries=150, sequenceid=173, filesize=11.9 K 2024-11-16T10:14:23,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/cf9aaa7a34ac4371822303d9480217e9 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/cf9aaa7a34ac4371822303d9480217e9 2024-11-16T10:14:23,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/cf9aaa7a34ac4371822303d9480217e9, entries=150, sequenceid=173, filesize=11.9 K 2024-11-16T10:14:23,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 2bab7dc9c814fe637b384a46040e48b5 in 674ms, sequenceid=173, compaction requested=true 2024-11-16T10:14:23,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:23,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:14:23,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:23,499 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:23,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:14:23,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:23,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:14:23,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T10:14:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:23,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-16T10:14:23,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:23,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:23,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:23,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:23,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:23,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:23,502 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:23,504 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38933 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:23,504 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/A is initiating minor compaction (all files) 2024-11-16T10:14:23,504 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:23,504 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/A in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:23,504 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/B is initiating minor compaction (all files) 2024-11-16T10:14:23,504 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/B in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:23,504 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/dd8c6aa16a5e418e9cfbc6587d333ee1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/1115ad2bb2534544b8661c0169c6ff31, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/c65f5bb7856d427596a33aa19a006ecc] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=38.0 K 2024-11-16T10:14:23,504 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/2663a08d1c454300a159faa8af6ceaf2, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/9b353d14d376437eb1e8fbf4ae64d39c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/c7576978ee234523a8abe3ef6e278c66] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=35.7 K 2024-11-16T10:14:23,505 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd8c6aa16a5e418e9cfbc6587d333ee1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731752061536 2024-11-16T10:14:23,505 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 2663a08d1c454300a159faa8af6ceaf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731752061536 2024-11-16T10:14:23,505 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b353d14d376437eb1e8fbf4ae64d39c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731752061633 2024-11-16T10:14:23,505 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1115ad2bb2534544b8661c0169c6ff31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731752061633 2024-11-16T10:14:23,506 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting c7576978ee234523a8abe3ef6e278c66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731752062268 2024-11-16T10:14:23,506 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting c65f5bb7856d427596a33aa19a006ecc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731752062268 2024-11-16T10:14:23,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/7302ca6703474608a88f4611cb73619c is 50, key is test_row_0/A:col10/1731752062912/Put/seqid=0 2024-11-16T10:14:23,527 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#B#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:23,527 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/101f8f8236654296bbec0ad06686af7c is 50, key is test_row_0/B:col10/1731752062300/Put/seqid=0 2024-11-16T10:14:23,529 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#A#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:23,530 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/01a1fd11b6fc4916b16dbc41c11462df is 50, key is test_row_0/A:col10/1731752062300/Put/seqid=0 2024-11-16T10:14:23,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752123533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:23,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741868_1044 (size=12493) 2024-11-16T10:14:23,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:23,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752123537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:23,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:23,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752123541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:23,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741867_1043 (size=12151) 2024-11-16T10:14:23,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741869_1045 (size=12493) 2024-11-16T10:14:23,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/7302ca6703474608a88f4611cb73619c 2024-11-16T10:14:23,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752123564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:23,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:23,570 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/01a1fd11b6fc4916b16dbc41c11462df as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/01a1fd11b6fc4916b16dbc41c11462df 2024-11-16T10:14:23,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752123574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:23,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/4c8772ff729c4bc7a8cfae3f4f7e18b5 is 50, key is test_row_0/B:col10/1731752062912/Put/seqid=0 2024-11-16T10:14:23,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:23,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T10:14:23,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:23,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:23,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:23,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:23,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:23,587 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/A of 2bab7dc9c814fe637b384a46040e48b5 into 01a1fd11b6fc4916b16dbc41c11462df(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:23,587 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:23,587 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/A, priority=13, startTime=1731752063499; duration=0sec 2024-11-16T10:14:23,587 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:23,587 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:A 2024-11-16T10:14:23,587 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:23,590 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:23,591 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/C is initiating minor compaction (all files) 2024-11-16T10:14:23,591 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/C in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:23,591 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/cd1878fea611407fb24c0d373ebb2977, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/6596d32419ea4269afbccbf8db8dd914, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/cf9aaa7a34ac4371822303d9480217e9] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=35.7 K 2024-11-16T10:14:23,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741870_1046 (size=12151) 2024-11-16T10:14:23,593 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd1878fea611407fb24c0d373ebb2977, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731752061536 2024-11-16T10:14:23,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/4c8772ff729c4bc7a8cfae3f4f7e18b5 2024-11-16T10:14:23,594 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6596d32419ea4269afbccbf8db8dd914, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731752061633 2024-11-16T10:14:23,594 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf9aaa7a34ac4371822303d9480217e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731752062268 2024-11-16T10:14:23,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/b3c421255ed947d0b65f7951d5d141ff is 50, key is test_row_0/C:col10/1731752062912/Put/seqid=0 2024-11-16T10:14:23,608 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#C#compaction#35 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:23,609 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/c8a029bb9d6346d3a3ebb1b6ed0da479 is 50, key is test_row_0/C:col10/1731752062300/Put/seqid=0 2024-11-16T10:14:23,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741871_1047 (size=12151) 2024-11-16T10:14:23,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/b3c421255ed947d0b65f7951d5d141ff 2024-11-16T10:14:23,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741872_1048 (size=12493) 2024-11-16T10:14:23,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/7302ca6703474608a88f4611cb73619c as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/7302ca6703474608a88f4611cb73619c 2024-11-16T10:14:23,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:23,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752123644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:23,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,651 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/c8a029bb9d6346d3a3ebb1b6ed0da479 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/c8a029bb9d6346d3a3ebb1b6ed0da479 2024-11-16T10:14:23,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:23,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/7302ca6703474608a88f4611cb73619c, entries=150, sequenceid=195, filesize=11.9 K 2024-11-16T10:14:23,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752123654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:23,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/4c8772ff729c4bc7a8cfae3f4f7e18b5 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4c8772ff729c4bc7a8cfae3f4f7e18b5 2024-11-16T10:14:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:23,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752123654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:23,672 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/C of 2bab7dc9c814fe637b384a46040e48b5 into c8a029bb9d6346d3a3ebb1b6ed0da479(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:23,672 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:23,672 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/C, priority=13, startTime=1731752063500; duration=0sec 2024-11-16T10:14:23,672 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:23,673 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:C 2024-11-16T10:14:23,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4c8772ff729c4bc7a8cfae3f4f7e18b5, entries=150, sequenceid=195, filesize=11.9 K 2024-11-16T10:14:23,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/b3c421255ed947d0b65f7951d5d141ff as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/b3c421255ed947d0b65f7951d5d141ff 2024-11-16T10:14:23,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/b3c421255ed947d0b65f7951d5d141ff, entries=150, sequenceid=195, filesize=11.9 K 2024-11-16T10:14:23,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 2bab7dc9c814fe637b384a46040e48b5 in 191ms, sequenceid=195, compaction requested=false 2024-11-16T10:14:23,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:23,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T10:14:23,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:23,736 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T10:14:23,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:23,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:23,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:23,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:23,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:23,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:23,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/44721fb754644a618b73ea89980e32ef is 50, key is test_row_0/A:col10/1731752063534/Put/seqid=0 2024-11-16T10:14:23,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741873_1049 (size=12151) 2024-11-16T10:14:23,765 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/44721fb754644a618b73ea89980e32ef 2024-11-16T10:14:23,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/a8c5607f98ae43899e762d2091e5267c is 50, key is test_row_0/B:col10/1731752063534/Put/seqid=0 2024-11-16T10:14:23,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741874_1050 (size=12151) 2024-11-16T10:14:23,792 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/a8c5607f98ae43899e762d2091e5267c 2024-11-16T10:14:23,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/1bc90786d8714b09bc8e1d88ac9b03d3 is 50, key is test_row_0/C:col10/1731752063534/Put/seqid=0 2024-11-16T10:14:23,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741875_1051 (size=12151) 2024-11-16T10:14:23,817 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/1bc90786d8714b09bc8e1d88ac9b03d3 2024-11-16T10:14:23,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/44721fb754644a618b73ea89980e32ef as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/44721fb754644a618b73ea89980e32ef 2024-11-16T10:14:23,839 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/44721fb754644a618b73ea89980e32ef, entries=150, sequenceid=207, filesize=11.9 K 2024-11-16T10:14:23,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/a8c5607f98ae43899e762d2091e5267c as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a8c5607f98ae43899e762d2091e5267c 2024-11-16T10:14:23,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:23,871 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a8c5607f98ae43899e762d2091e5267c, entries=150, sequenceid=207, filesize=11.9 K 2024-11-16T10:14:23,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/1bc90786d8714b09bc8e1d88ac9b03d3 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/1bc90786d8714b09bc8e1d88ac9b03d3 2024-11-16T10:14:23,885 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/1bc90786d8714b09bc8e1d88ac9b03d3, entries=150, sequenceid=207, filesize=11.9 K 2024-11-16T10:14:23,887 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=67.09 KB/68700 for 2bab7dc9c814fe637b384a46040e48b5 in 151ms, sequenceid=207, compaction requested=true 2024-11-16T10:14:23,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:23,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:23,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-16T10:14:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-16T10:14:23,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T10:14:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:23,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-16T10:14:23,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0900 sec 2024-11-16T10:14:23,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/d68479c495034b459b3b68c1fe809561 is 50, key is test_row_0/A:col10/1731752063880/Put/seqid=0 2024-11-16T10:14:23,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.1150 sec 2024-11-16T10:14:23,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741876_1052 (size=14541) 2024-11-16T10:14:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T10:14:23,934 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:14:23,935 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:14:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-16T10:14:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T10:14:23,938 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:14:23,939 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:14:23,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:14:23,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752123932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:23,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752123932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:23,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:23,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:23,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752123933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:23,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:23,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:23,952 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/101f8f8236654296bbec0ad06686af7c as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/101f8f8236654296bbec0ad06686af7c 2024-11-16T10:14:23,963 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/B of 2bab7dc9c814fe637b384a46040e48b5 into 101f8f8236654296bbec0ad06686af7c(size=12.2 K), total size for store is 35.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:23,963 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:23,963 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/B, priority=13, startTime=1731752063499; duration=0sec 2024-11-16T10:14:23,963 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:23,963 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:B 2024-11-16T10:14:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T10:14:24,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752124045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:24,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:24,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:24,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:24,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752124045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:24,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:24,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:24,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:24,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752124046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:24,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:24,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:24,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:24,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752124074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:24,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:24,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:24,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:24,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T10:14:24,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:24,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752124094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:24,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:24,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:24,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:24,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:24,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:24,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:24,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:24,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T10:14:24,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:24,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:24,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:24,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:24,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:24,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T10:14:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:24,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:24,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752124254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:24,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:24,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:24,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:24,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:24,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752124254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752124255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:24,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:24,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:24,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:24,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/d68479c495034b459b3b68c1fe809561 2024-11-16T10:14:24,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/9f3549a2a57a43e6aeb58a4a18a7ba12 is 50, key is test_row_0/B:col10/1731752063880/Put/seqid=0 2024-11-16T10:14:24,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741877_1053 (size=12151) 2024-11-16T10:14:24,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/9f3549a2a57a43e6aeb58a4a18a7ba12 2024-11-16T10:14:24,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/29d6cfc16f75456ca64f074079442fab is 50, key is test_row_0/C:col10/1731752063880/Put/seqid=0 2024-11-16T10:14:24,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741878_1054 (size=12151) 2024-11-16T10:14:24,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T10:14:24,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:24,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:24,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:24,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:24,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:24,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T10:14:24,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:24,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:24,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:24,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:24,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:24,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T10:14:24,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752124565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:24,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:24,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:24,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:24,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752124565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:24,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:24,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:24,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752124565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:24,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:24,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T10:14:24,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:24,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:24,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:24,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:24,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:24,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/29d6cfc16f75456ca64f074079442fab 2024-11-16T10:14:24,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/d68479c495034b459b3b68c1fe809561 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/d68479c495034b459b3b68c1fe809561 2024-11-16T10:14:24,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/d68479c495034b459b3b68c1fe809561, entries=200, sequenceid=221, filesize=14.2 K 2024-11-16T10:14:24,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/9f3549a2a57a43e6aeb58a4a18a7ba12 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/9f3549a2a57a43e6aeb58a4a18a7ba12 2024-11-16T10:14:24,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/9f3549a2a57a43e6aeb58a4a18a7ba12, entries=150, sequenceid=221, filesize=11.9 K 2024-11-16T10:14:24,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/29d6cfc16f75456ca64f074079442fab as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/29d6cfc16f75456ca64f074079442fab 2024-11-16T10:14:24,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/29d6cfc16f75456ca64f074079442fab, entries=150, sequenceid=221, filesize=11.9 K 2024-11-16T10:14:24,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 2bab7dc9c814fe637b384a46040e48b5 in 911ms, sequenceid=221, compaction requested=true 2024-11-16T10:14:24,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:24,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:14:24,802 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:14:24,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:24,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:14:24,802 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:14:24,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:24,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:14:24,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:24,803 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51336 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:14:24,804 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/A is initiating minor compaction (all files) 2024-11-16T10:14:24,805 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:14:24,805 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/B is initiating minor compaction (all files) 2024-11-16T10:14:24,805 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/B in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:24,805 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/101f8f8236654296bbec0ad06686af7c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4c8772ff729c4bc7a8cfae3f4f7e18b5, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a8c5607f98ae43899e762d2091e5267c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/9f3549a2a57a43e6aeb58a4a18a7ba12] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=47.8 K 2024-11-16T10:14:24,806 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 101f8f8236654296bbec0ad06686af7c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731752062268 2024-11-16T10:14:24,806 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c8772ff729c4bc7a8cfae3f4f7e18b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731752062912 2024-11-16T10:14:24,806 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting a8c5607f98ae43899e762d2091e5267c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731752063519 2024-11-16T10:14:24,807 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f3549a2a57a43e6aeb58a4a18a7ba12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731752063880 2024-11-16T10:14:24,804 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/A in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:24,808 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/01a1fd11b6fc4916b16dbc41c11462df, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/7302ca6703474608a88f4611cb73619c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/44721fb754644a618b73ea89980e32ef, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/d68479c495034b459b3b68c1fe809561] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=50.1 K 2024-11-16T10:14:24,809 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01a1fd11b6fc4916b16dbc41c11462df, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731752062268 2024-11-16T10:14:24,809 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7302ca6703474608a88f4611cb73619c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731752062912 2024-11-16T10:14:24,810 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44721fb754644a618b73ea89980e32ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731752063519 2024-11-16T10:14:24,810 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting d68479c495034b459b3b68c1fe809561, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731752063865 2024-11-16T10:14:24,828 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#B#compaction#42 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:24,829 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/4897668181194e71951b92497f27f7de is 50, key is test_row_0/B:col10/1731752063880/Put/seqid=0 2024-11-16T10:14:24,835 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#A#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:24,836 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/64db3a91f8cf4f7883d63e414b35cc96 is 50, key is test_row_0/A:col10/1731752063880/Put/seqid=0 2024-11-16T10:14:24,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741879_1055 (size=12629) 2024-11-16T10:14:24,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741880_1056 (size=12629) 2024-11-16T10:14:24,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T10:14:24,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:24,870 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-16T10:14:24,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:24,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:24,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:24,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:24,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:24,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:24,881 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/64db3a91f8cf4f7883d63e414b35cc96 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/64db3a91f8cf4f7883d63e414b35cc96 2024-11-16T10:14:24,881 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/4897668181194e71951b92497f27f7de as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4897668181194e71951b92497f27f7de 2024-11-16T10:14:24,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/10e40e471fd24da887d6c07c515ae15c is 50, key is test_row_0/A:col10/1731752063931/Put/seqid=0 2024-11-16T10:14:24,894 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/B of 2bab7dc9c814fe637b384a46040e48b5 into 4897668181194e71951b92497f27f7de(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:24,894 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:24,894 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/B, priority=12, startTime=1731752064802; duration=0sec 2024-11-16T10:14:24,895 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:24,895 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:B 2024-11-16T10:14:24,895 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:14:24,896 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/A of 2bab7dc9c814fe637b384a46040e48b5 into 64db3a91f8cf4f7883d63e414b35cc96(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:24,896 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:24,896 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/A, priority=12, startTime=1731752064802; duration=0sec 2024-11-16T10:14:24,896 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:24,896 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:A 2024-11-16T10:14:24,897 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:14:24,898 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/C is initiating minor compaction (all files) 2024-11-16T10:14:24,898 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/C in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:24,898 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/c8a029bb9d6346d3a3ebb1b6ed0da479, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/b3c421255ed947d0b65f7951d5d141ff, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/1bc90786d8714b09bc8e1d88ac9b03d3, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/29d6cfc16f75456ca64f074079442fab] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=47.8 K 2024-11-16T10:14:24,899 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting c8a029bb9d6346d3a3ebb1b6ed0da479, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731752062268 2024-11-16T10:14:24,900 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting b3c421255ed947d0b65f7951d5d141ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731752062912 2024-11-16T10:14:24,901 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bc90786d8714b09bc8e1d88ac9b03d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731752063519 2024-11-16T10:14:24,901 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 29d6cfc16f75456ca64f074079442fab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731752063880 2024-11-16T10:14:24,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741881_1057 (size=12151) 2024-11-16T10:14:24,927 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/10e40e471fd24da887d6c07c515ae15c 2024-11-16T10:14:24,935 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#C#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:24,936 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/d2c03aab8e4d43ebbfa23c8ccfc30198 is 50, key is test_row_0/C:col10/1731752063880/Put/seqid=0 2024-11-16T10:14:24,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/6ef08f67c7704da4afbdb13c8515d5b4 is 50, key is test_row_0/B:col10/1731752063931/Put/seqid=0 2024-11-16T10:14:24,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741882_1058 (size=12629) 2024-11-16T10:14:24,969 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/d2c03aab8e4d43ebbfa23c8ccfc30198 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/d2c03aab8e4d43ebbfa23c8ccfc30198 2024-11-16T10:14:24,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741883_1059 (size=12151) 2024-11-16T10:14:24,972 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/6ef08f67c7704da4afbdb13c8515d5b4 2024-11-16T10:14:24,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/a9110092f80e4c0b9aa1afbed8a0853a is 50, key is test_row_0/C:col10/1731752063931/Put/seqid=0 2024-11-16T10:14:24,985 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/C of 2bab7dc9c814fe637b384a46040e48b5 into d2c03aab8e4d43ebbfa23c8ccfc30198(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:24,985 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:24,985 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/C, priority=12, startTime=1731752064802; duration=0sec 2024-11-16T10:14:24,986 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:24,986 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:C 2024-11-16T10:14:24,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741884_1060 (size=12151) 2024-11-16T10:14:24,997 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/a9110092f80e4c0b9aa1afbed8a0853a 2024-11-16T10:14:25,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/10e40e471fd24da887d6c07c515ae15c as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/10e40e471fd24da887d6c07c515ae15c 2024-11-16T10:14:25,026 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/10e40e471fd24da887d6c07c515ae15c, entries=150, sequenceid=240, filesize=11.9 K 2024-11-16T10:14:25,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/6ef08f67c7704da4afbdb13c8515d5b4 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6ef08f67c7704da4afbdb13c8515d5b4 2024-11-16T10:14:25,035 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6ef08f67c7704da4afbdb13c8515d5b4, entries=150, sequenceid=240, filesize=11.9 K 2024-11-16T10:14:25,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/a9110092f80e4c0b9aa1afbed8a0853a as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/a9110092f80e4c0b9aa1afbed8a0853a 2024-11-16T10:14:25,046 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/a9110092f80e4c0b9aa1afbed8a0853a, entries=150, sequenceid=240, filesize=11.9 K 2024-11-16T10:14:25,048 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 2bab7dc9c814fe637b384a46040e48b5 in 179ms, sequenceid=240, compaction requested=false 2024-11-16T10:14:25,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:25,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:25,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-16T10:14:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-16T10:14:25,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-16T10:14:25,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1110 sec 2024-11-16T10:14:25,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.1200 sec 2024-11-16T10:14:25,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T10:14:25,074 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:14:25,075 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:14:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-16T10:14:25,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T10:14:25,078 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:14:25,080 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:14:25,081 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:14:25,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:25,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T10:14:25,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:25,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:25,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:25,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:25,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:25,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:25,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/c10438f825cc46ce8b9fc54dd453e5f0 is 50, key is test_row_0/A:col10/1731752065086/Put/seqid=0 2024-11-16T10:14:25,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741885_1061 (size=14541) 2024-11-16T10:14:25,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/c10438f825cc46ce8b9fc54dd453e5f0 2024-11-16T10:14:25,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752125131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752065127, max timestamp=1731752065138Num uniques 859; 2024-11-16T10:14:25,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752065127, max timestamp=1731752065138Num uniques 859; 2024-11-16T10:14:25,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752065127, max timestamp=1731752065138Num uniques 859; 2024-11-16T10:14:25,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:14:25,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:14:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:14:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:25,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/322e3908a6cc4d22b992e7e2ac1976ec is 50, key is test_row_0/B:col10/1731752065086/Put/seqid=0 2024-11-16T10:14:25,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741886_1062 (size=12151) 2024-11-16T10:14:25,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/322e3908a6cc4d22b992e7e2ac1976ec 2024-11-16T10:14:25,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752125168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752125170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752125171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:25,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752125176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T10:14:25,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/f6402589a8874ee8b000e8628aa141ee is 50, key is test_row_0/C:col10/1731752065086/Put/seqid=0 2024-11-16T10:14:25,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741887_1063 (size=12151) 2024-11-16T10:14:25,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/f6402589a8874ee8b000e8628aa141ee 2024-11-16T10:14:25,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/c10438f825cc46ce8b9fc54dd453e5f0 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/c10438f825cc46ce8b9fc54dd453e5f0 2024-11-16T10:14:25,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T10:14:25,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:25,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/c10438f825cc46ce8b9fc54dd453e5f0, entries=200, sequenceid=254, filesize=14.2 K 2024-11-16T10:14:25,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:25,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:25,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:25,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:25,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:25,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/322e3908a6cc4d22b992e7e2ac1976ec as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/322e3908a6cc4d22b992e7e2ac1976ec 2024-11-16T10:14:25,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/322e3908a6cc4d22b992e7e2ac1976ec, entries=150, sequenceid=254, filesize=11.9 K 2024-11-16T10:14:25,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/f6402589a8874ee8b000e8628aa141ee as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/f6402589a8874ee8b000e8628aa141ee 2024-11-16T10:14:25,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/f6402589a8874ee8b000e8628aa141ee, entries=150, sequenceid=254, filesize=11.9 K 2024-11-16T10:14:25,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 2bab7dc9c814fe637b384a46040e48b5 in 167ms, sequenceid=254, compaction requested=true 2024-11-16T10:14:25,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:25,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:14:25,257 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:25,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:25,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:14:25,257 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:25,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:25,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:14:25,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:25,259 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:25,259 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/B is initiating minor compaction (all files) 2024-11-16T10:14:25,259 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/B in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:25,259 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4897668181194e71951b92497f27f7de, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6ef08f67c7704da4afbdb13c8515d5b4, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/322e3908a6cc4d22b992e7e2ac1976ec] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=36.1 K 2024-11-16T10:14:25,260 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:25,260 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/A is initiating minor compaction (all files) 2024-11-16T10:14:25,260 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/A in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:25,261 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/64db3a91f8cf4f7883d63e414b35cc96, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/10e40e471fd24da887d6c07c515ae15c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/c10438f825cc46ce8b9fc54dd453e5f0] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=38.4 K 2024-11-16T10:14:25,262 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 4897668181194e71951b92497f27f7de, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731752063880 2024-11-16T10:14:25,262 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64db3a91f8cf4f7883d63e414b35cc96, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731752063880 2024-11-16T10:14:25,262 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ef08f67c7704da4afbdb13c8515d5b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731752063919 2024-11-16T10:14:25,263 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10e40e471fd24da887d6c07c515ae15c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731752063919 2024-11-16T10:14:25,263 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 322e3908a6cc4d22b992e7e2ac1976ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731752065084 2024-11-16T10:14:25,263 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting c10438f825cc46ce8b9fc54dd453e5f0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731752065084 2024-11-16T10:14:25,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:25,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-16T10:14:25,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:25,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T10:14:25,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:25,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T10:14:25,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:25,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T10:14:25,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/2351409015d942299168b37daf81fbe7 is 50, key is test_row_0/A:col10/1731752065271/Put/seqid=0 2024-11-16T10:14:25,286 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#B#compaction#53 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:25,287 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/6e5a9f56285b4cad9109d530674382fb is 50, key is test_row_0/B:col10/1731752065086/Put/seqid=0 2024-11-16T10:14:25,289 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#A#compaction#51 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:25,290 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/33da2c5562f3462d9d10af4aadc0e975 is 50, key is test_row_0/A:col10/1731752065086/Put/seqid=0 2024-11-16T10:14:25,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741889_1065 (size=12731) 2024-11-16T10:14:25,305 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/6e5a9f56285b4cad9109d530674382fb as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6e5a9f56285b4cad9109d530674382fb 2024-11-16T10:14:25,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752125293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:25,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752125295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752125296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:25,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752125299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752125306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:25,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:25,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741890_1066 (size=12731) 2024-11-16T10:14:25,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741888_1064 (size=14741) 2024-11-16T10:14:25,318 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/B of 2bab7dc9c814fe637b384a46040e48b5 into 6e5a9f56285b4cad9109d530674382fb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:25,318 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:25,318 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/B, priority=13, startTime=1731752065257; duration=0sec 2024-11-16T10:14:25,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/2351409015d942299168b37daf81fbe7 2024-11-16T10:14:25,318 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:25,318 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:B 2024-11-16T10:14:25,318 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:25,324 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/33da2c5562f3462d9d10af4aadc0e975 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/33da2c5562f3462d9d10af4aadc0e975 2024-11-16T10:14:25,325 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:25,325 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/C is initiating minor compaction (all files) 2024-11-16T10:14:25,327 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/C in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:25,327 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/d2c03aab8e4d43ebbfa23c8ccfc30198, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/a9110092f80e4c0b9aa1afbed8a0853a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/f6402589a8874ee8b000e8628aa141ee] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=36.1 K 2024-11-16T10:14:25,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/ac5b24185f0744abb9f8c97d5dc83a55 is 50, key is test_row_0/B:col10/1731752065271/Put/seqid=0 2024-11-16T10:14:25,332 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting d2c03aab8e4d43ebbfa23c8ccfc30198, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731752063880 2024-11-16T10:14:25,333 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting a9110092f80e4c0b9aa1afbed8a0853a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731752063919 2024-11-16T10:14:25,333 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting f6402589a8874ee8b000e8628aa141ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731752065084 2024-11-16T10:14:25,335 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/A of 2bab7dc9c814fe637b384a46040e48b5 into 33da2c5562f3462d9d10af4aadc0e975(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:25,335 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:25,335 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/A, priority=13, startTime=1731752065257; duration=0sec 2024-11-16T10:14:25,335 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:25,335 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:A 2024-11-16T10:14:25,346 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#C#compaction#55 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:25,346 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/1a13375fe32e4019aca5847eef2f0f03 is 50, key is test_row_0/C:col10/1731752065086/Put/seqid=0 2024-11-16T10:14:25,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741891_1067 (size=12301) 2024-11-16T10:14:25,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/ac5b24185f0744abb9f8c97d5dc83a55 2024-11-16T10:14:25,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741892_1068 (size=12731) 2024-11-16T10:14:25,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/13b25096a7934a6f8246c51b4b44f190 is 50, key is test_row_0/C:col10/1731752065271/Put/seqid=0 2024-11-16T10:14:25,365 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/1a13375fe32e4019aca5847eef2f0f03 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/1a13375fe32e4019aca5847eef2f0f03 2024-11-16T10:14:25,374 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/C of 2bab7dc9c814fe637b384a46040e48b5 into 1a13375fe32e4019aca5847eef2f0f03(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:25,374 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:25,374 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/C, priority=13, startTime=1731752065257; duration=0sec 2024-11-16T10:14:25,374 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:25,374 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:C 2024-11-16T10:14:25,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741893_1069 (size=12301) 2024-11-16T10:14:25,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T10:14:25,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:25,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:25,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:25,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:25,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:25,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T10:14:25,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752125414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752125414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:25,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752125414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:25,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752125414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:25,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752125515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:25,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T10:14:25,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:25,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:25,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:25,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:25,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:25,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752125624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752125624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:25,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752125624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:25,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752125625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:25,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T10:14:25,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:25,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:25,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:25,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:25,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T10:14:25,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/13b25096a7934a6f8246c51b4b44f190 2024-11-16T10:14:25,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/2351409015d942299168b37daf81fbe7 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/2351409015d942299168b37daf81fbe7 2024-11-16T10:14:25,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/2351409015d942299168b37daf81fbe7, entries=200, sequenceid=292, filesize=14.4 K 2024-11-16T10:14:25,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/ac5b24185f0744abb9f8c97d5dc83a55 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/ac5b24185f0744abb9f8c97d5dc83a55 2024-11-16T10:14:25,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/ac5b24185f0744abb9f8c97d5dc83a55, entries=150, sequenceid=292, filesize=12.0 K 2024-11-16T10:14:25,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/13b25096a7934a6f8246c51b4b44f190 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/13b25096a7934a6f8246c51b4b44f190 2024-11-16T10:14:25,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/13b25096a7934a6f8246c51b4b44f190, entries=150, sequenceid=292, filesize=12.0 K 2024-11-16T10:14:25,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 2bab7dc9c814fe637b384a46040e48b5 in 544ms, sequenceid=292, compaction requested=false 2024-11-16T10:14:25,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:25,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:25,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T10:14:25,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:25,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:25,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:25,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:25,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:25,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:25,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T10:14:25,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:25,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:25,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:25,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:25,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:25,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/ecc9da2ff0cb4caa99ad81d82e277eab is 50, key is test_row_0/A:col10/1731752065836/Put/seqid=0 2024-11-16T10:14:25,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741894_1070 (size=12301) 2024-11-16T10:14:25,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/ecc9da2ff0cb4caa99ad81d82e277eab 2024-11-16T10:14:25,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/9f3dfab7b31046ebbe5596855dbbf5b6 is 50, key is test_row_0/B:col10/1731752065836/Put/seqid=0 2024-11-16T10:14:25,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741895_1071 (size=12301) 2024-11-16T10:14:25,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731752065912, max timestamp=1731752065944Num uniques 859; 2024-11-16T10:14:25,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731752065912, max timestamp=1731752065944Num uniques 859; 2024-11-16T10:14:25,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731752065912, max timestamp=1731752065944Num uniques 859; 2024-11-16T10:14:25,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-16T10:14:25,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-16T10:14:25,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-16T10:14:25,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752125960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:25,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752125961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:25,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752125964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:25,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752125965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:25,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:25,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752125967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:25,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:25,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:26,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T10:14:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:26,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:26,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:26,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752126074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752126074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:26,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752126074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752126075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:26,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752126084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:26,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T10:14:26,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:26,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:26,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:26,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:26,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T10:14:26,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752126284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:26,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:26,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752126284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:26,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:26,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752126284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:26,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:26,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752126284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:26,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:26,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752126294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:26,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:26,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/9f3dfab7b31046ebbe5596855dbbf5b6 2024-11-16T10:14:26,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T10:14:26,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:26,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:26,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:26,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:26,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:26,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/fafef004ca4e43528c432cc760c2558e is 50, key is test_row_0/C:col10/1731752065836/Put/seqid=0 2024-11-16T10:14:26,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741896_1072 (size=12301) 2024-11-16T10:14:26,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T10:14:26,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:26,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:26,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:26,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:26,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:26,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:26,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752126594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752126594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:26,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:26,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:26,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752126595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752126595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:26,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:26,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752126604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:26,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T10:14:26,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:26,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:26,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:26,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:26,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:26,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/fafef004ca4e43528c432cc760c2558e 2024-11-16T10:14:26,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/ecc9da2ff0cb4caa99ad81d82e277eab as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/ecc9da2ff0cb4caa99ad81d82e277eab 2024-11-16T10:14:26,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/ecc9da2ff0cb4caa99ad81d82e277eab, entries=150, sequenceid=305, filesize=12.0 K 2024-11-16T10:14:26,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/9f3dfab7b31046ebbe5596855dbbf5b6 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/9f3dfab7b31046ebbe5596855dbbf5b6 2024-11-16T10:14:26,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/9f3dfab7b31046ebbe5596855dbbf5b6, entries=150, sequenceid=305, filesize=12.0 K 2024-11-16T10:14:26,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/fafef004ca4e43528c432cc760c2558e as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/fafef004ca4e43528c432cc760c2558e 2024-11-16T10:14:26,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/fafef004ca4e43528c432cc760c2558e, entries=150, sequenceid=305, filesize=12.0 K 2024-11-16T10:14:26,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 2bab7dc9c814fe637b384a46040e48b5 in 915ms, sequenceid=305, compaction requested=true 2024-11-16T10:14:26,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:26,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:14:26,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:26,758 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:26,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:14:26,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:26,758 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:26,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:14:26,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:26,759 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:26,759 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/A is initiating minor compaction (all files) 2024-11-16T10:14:26,759 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/A in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:26,760 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/33da2c5562f3462d9d10af4aadc0e975, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/2351409015d942299168b37daf81fbe7, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/ecc9da2ff0cb4caa99ad81d82e277eab] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=38.8 K 2024-11-16T10:14:26,760 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:26,760 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33da2c5562f3462d9d10af4aadc0e975, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731752065084 2024-11-16T10:14:26,760 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/B is initiating minor compaction (all files) 2024-11-16T10:14:26,760 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/B in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:26,760 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6e5a9f56285b4cad9109d530674382fb, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/ac5b24185f0744abb9f8c97d5dc83a55, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/9f3dfab7b31046ebbe5596855dbbf5b6] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=36.5 K 2024-11-16T10:14:26,761 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e5a9f56285b4cad9109d530674382fb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731752065084 2024-11-16T10:14:26,762 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting ac5b24185f0744abb9f8c97d5dc83a55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731752065167 2024-11-16T10:14:26,762 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f3dfab7b31046ebbe5596855dbbf5b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731752065296 2024-11-16T10:14:26,763 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2351409015d942299168b37daf81fbe7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731752065167 2024-11-16T10:14:26,764 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecc9da2ff0cb4caa99ad81d82e277eab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731752065296 2024-11-16T10:14:26,772 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#B#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:26,773 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/61ec9b01c0ef4a25a9201f7a5bdf8b15 is 50, key is test_row_0/B:col10/1731752065836/Put/seqid=0 2024-11-16T10:14:26,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T10:14:26,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:26,773 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#A#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:26,773 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=140.89 KB heapSize=381.47 KB 2024-11-16T10:14:26,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:26,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:26,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:26,774 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/4b716fb13c8a4241b89ab037a62142ae is 50, key is test_row_0/A:col10/1731752065836/Put/seqid=0 2024-11-16T10:14:26,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:26,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:26,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:26,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/81d8f57231344d7e9d843a62a7298435 is 50, key is test_row_0/A:col10/1731752065958/Put/seqid=0 2024-11-16T10:14:26,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741898_1074 (size=12983) 2024-11-16T10:14:26,799 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/4b716fb13c8a4241b89ab037a62142ae as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/4b716fb13c8a4241b89ab037a62142ae 2024-11-16T10:14:26,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741897_1073 (size=12983) 2024-11-16T10:14:26,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741899_1075 (size=12301) 2024-11-16T10:14:26,804 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/81d8f57231344d7e9d843a62a7298435 2024-11-16T10:14:26,809 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/61ec9b01c0ef4a25a9201f7a5bdf8b15 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/61ec9b01c0ef4a25a9201f7a5bdf8b15 2024-11-16T10:14:26,810 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/A of 2bab7dc9c814fe637b384a46040e48b5 into 4b716fb13c8a4241b89ab037a62142ae(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:26,810 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:26,810 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/A, priority=13, startTime=1731752066758; duration=0sec 2024-11-16T10:14:26,810 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:26,810 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:A 2024-11-16T10:14:26,810 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:26,813 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:26,813 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/C is initiating minor compaction (all files) 2024-11-16T10:14:26,813 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/C in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:26,814 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/1a13375fe32e4019aca5847eef2f0f03, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/13b25096a7934a6f8246c51b4b44f190, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/fafef004ca4e43528c432cc760c2558e] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=36.5 K 2024-11-16T10:14:26,814 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a13375fe32e4019aca5847eef2f0f03, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731752065084 2024-11-16T10:14:26,815 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13b25096a7934a6f8246c51b4b44f190, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731752065167 2024-11-16T10:14:26,816 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting fafef004ca4e43528c432cc760c2558e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731752065296 2024-11-16T10:14:26,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/17c127d8914b4198886e621e99afc575 is 50, key is test_row_0/B:col10/1731752065958/Put/seqid=0 2024-11-16T10:14:26,824 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/B of 2bab7dc9c814fe637b384a46040e48b5 into 61ec9b01c0ef4a25a9201f7a5bdf8b15(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:26,824 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:26,824 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/B, priority=13, startTime=1731752066758; duration=0sec 2024-11-16T10:14:26,824 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:26,824 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:B 2024-11-16T10:14:26,842 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#C#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:26,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741900_1076 (size=12301) 2024-11-16T10:14:26,842 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/58807ce42796402ab38885b243203804 is 50, key is test_row_0/C:col10/1731752065836/Put/seqid=0 2024-11-16T10:14:26,844 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/17c127d8914b4198886e621e99afc575 2024-11-16T10:14:26,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741901_1077 (size=12983) 2024-11-16T10:14:26,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/4784b3797cb447b4ba8038dd79dbe68c is 50, key is test_row_0/C:col10/1731752065958/Put/seqid=0 2024-11-16T10:14:26,876 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/58807ce42796402ab38885b243203804 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/58807ce42796402ab38885b243203804 2024-11-16T10:14:26,888 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/C of 2bab7dc9c814fe637b384a46040e48b5 into 58807ce42796402ab38885b243203804(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:26,888 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:26,888 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/C, priority=13, startTime=1731752066758; duration=0sec 2024-11-16T10:14:26,889 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:26,889 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:C 2024-11-16T10:14:26,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741902_1078 (size=12301) 2024-11-16T10:14:26,891 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/4784b3797cb447b4ba8038dd79dbe68c 2024-11-16T10:14:26,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/81d8f57231344d7e9d843a62a7298435 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/81d8f57231344d7e9d843a62a7298435 2024-11-16T10:14:26,911 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/81d8f57231344d7e9d843a62a7298435, entries=150, sequenceid=338, filesize=12.0 K 2024-11-16T10:14:26,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/17c127d8914b4198886e621e99afc575 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/17c127d8914b4198886e621e99afc575 2024-11-16T10:14:26,928 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/17c127d8914b4198886e621e99afc575, entries=150, sequenceid=338, filesize=12.0 K 2024-11-16T10:14:26,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/4784b3797cb447b4ba8038dd79dbe68c as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/4784b3797cb447b4ba8038dd79dbe68c 2024-11-16T10:14:26,939 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/4784b3797cb447b4ba8038dd79dbe68c, entries=150, sequenceid=338, filesize=12.0 K 2024-11-16T10:14:26,940 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~381.42 KB/390576, currentSize=0 B/0 for 2bab7dc9c814fe637b384a46040e48b5 in 167ms, sequenceid=338, compaction requested=false 2024-11-16T10:14:26,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:26,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:26,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-16T10:14:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-16T10:14:26,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-16T10:14:26,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8610 sec 2024-11-16T10:14:26,946 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.8690 sec 2024-11-16T10:14:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:27,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T10:14:27,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:27,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:27,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:27,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:27,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:27,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:27,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/531ce653a2224b40a0c43beae26ba4ef is 50, key is test_row_0/A:col10/1731752067115/Put/seqid=0 2024-11-16T10:14:27,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741903_1079 (size=14741) 2024-11-16T10:14:27,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731752067144, max timestamp=1731752067148Num uniques 859; 2024-11-16T10:14:27,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752067144, max timestamp=1731752067148Num uniques 859; 2024-11-16T10:14:27,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:14:27,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:14:27,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752067144, max timestamp=1731752067148Num uniques 859; 2024-11-16T10:14:27,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:14:27,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752127172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:27,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:27,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752127173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:27,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:27,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:27,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752127174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:27,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752127183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:27,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752127183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T10:14:27,234 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:14:27,235 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:14:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-16T10:14:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T10:14:27,238 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:14:27,239 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:14:27,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:14:27,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752127295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:27,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752127295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752127295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:27,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752127295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752127295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:27,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:27,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T10:14:27,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T10:14:27,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:27,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:27,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:27,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:27,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:27,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752127504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:27,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:27,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752127514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752127515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752127515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752127515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:27,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:27,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/531ce653a2224b40a0c43beae26ba4ef 2024-11-16T10:14:27,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/5b1aaeb0ed3c49f0b5bca761b65519ff is 50, key is test_row_0/B:col10/1731752067115/Put/seqid=0 2024-11-16T10:14:27,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T10:14:27,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:27,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:27,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:27,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:27,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:27,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741904_1080 (size=12301) 2024-11-16T10:14:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T10:14:27,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T10:14:27,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:27,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:27,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:27,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:27,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:27,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752127815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:27,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:27,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752127836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:27,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752127836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:27,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752127842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:27,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752127842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:27,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:27,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:27,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T10:14:27,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:27,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:27,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:27,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:27,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T10:14:27,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/5b1aaeb0ed3c49f0b5bca761b65519ff 2024-11-16T10:14:27,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/3b5f52bcac08446d96ca33fac531241d is 50, key is test_row_0/C:col10/1731752067115/Put/seqid=0 2024-11-16T10:14:27,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741905_1081 (size=12301) 2024-11-16T10:14:28,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T10:14:28,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:28,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:28,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:28,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:28,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:28,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T10:14:28,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:28,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:28,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:28,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:28,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:28,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T10:14:28,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:28,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:28,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:28,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:28,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:28,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752128324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:28,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:28,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752128355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:28,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:28,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:28,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752128355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:28,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:28,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:28,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:28,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:28,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752128355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:28,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752128355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:28,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/3b5f52bcac08446d96ca33fac531241d 2024-11-16T10:14:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T10:14:28,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/531ce653a2224b40a0c43beae26ba4ef as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/531ce653a2224b40a0c43beae26ba4ef 2024-11-16T10:14:28,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/531ce653a2224b40a0c43beae26ba4ef, entries=200, sequenceid=352, filesize=14.4 K 2024-11-16T10:14:28,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/5b1aaeb0ed3c49f0b5bca761b65519ff as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/5b1aaeb0ed3c49f0b5bca761b65519ff 2024-11-16T10:14:28,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/5b1aaeb0ed3c49f0b5bca761b65519ff, entries=150, sequenceid=352, filesize=12.0 K 2024-11-16T10:14:28,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/3b5f52bcac08446d96ca33fac531241d as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/3b5f52bcac08446d96ca33fac531241d 2024-11-16T10:14:28,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/3b5f52bcac08446d96ca33fac531241d, entries=150, sequenceid=352, filesize=12.0 K 2024-11-16T10:14:28,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 2bab7dc9c814fe637b384a46040e48b5 in 1290ms, sequenceid=352, compaction requested=true 2024-11-16T10:14:28,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:28,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:14:28,406 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:28,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:28,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:14:28,407 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:28,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:28,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:14:28,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:28,408 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:28,408 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:28,408 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/B is initiating minor compaction (all files) 2024-11-16T10:14:28,408 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/A is initiating minor compaction (all files) 2024-11-16T10:14:28,408 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/B in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:28,408 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/A in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:28,408 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/61ec9b01c0ef4a25a9201f7a5bdf8b15, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/17c127d8914b4198886e621e99afc575, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/5b1aaeb0ed3c49f0b5bca761b65519ff] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=36.7 K 2024-11-16T10:14:28,408 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/4b716fb13c8a4241b89ab037a62142ae, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/81d8f57231344d7e9d843a62a7298435, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/531ce653a2224b40a0c43beae26ba4ef] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=39.1 K 2024-11-16T10:14:28,409 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b716fb13c8a4241b89ab037a62142ae, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731752065296 2024-11-16T10:14:28,409 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 61ec9b01c0ef4a25a9201f7a5bdf8b15, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731752065296 2024-11-16T10:14:28,410 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 17c127d8914b4198886e621e99afc575, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731752065958 2024-11-16T10:14:28,410 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81d8f57231344d7e9d843a62a7298435, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731752065958 2024-11-16T10:14:28,410 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b1aaeb0ed3c49f0b5bca761b65519ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731752067112 2024-11-16T10:14:28,410 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 531ce653a2224b40a0c43beae26ba4ef, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731752067112 2024-11-16T10:14:28,424 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#B#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:28,424 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/008a4a42e20c4934acd82ae31692e312 is 50, key is test_row_0/B:col10/1731752067115/Put/seqid=0 2024-11-16T10:14:28,430 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#A#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:28,431 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/ec32153741b447d9a8b9168301e84c1b is 50, key is test_row_0/A:col10/1731752067115/Put/seqid=0 2024-11-16T10:14:28,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741906_1082 (size=13085) 2024-11-16T10:14:28,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T10:14:28,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:28,469 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-16T10:14:28,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:28,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:28,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:28,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:28,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:28,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:28,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741907_1083 (size=13085) 2024-11-16T10:14:28,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/cffabf1318e840e4b655bd650af85a36 is 50, key is test_row_0/A:col10/1731752067180/Put/seqid=0 2024-11-16T10:14:28,519 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/ec32153741b447d9a8b9168301e84c1b as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/ec32153741b447d9a8b9168301e84c1b 2024-11-16T10:14:28,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741908_1084 (size=12301) 2024-11-16T10:14:28,528 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/cffabf1318e840e4b655bd650af85a36 2024-11-16T10:14:28,528 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/A of 2bab7dc9c814fe637b384a46040e48b5 into ec32153741b447d9a8b9168301e84c1b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:28,529 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:28,529 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/A, priority=13, startTime=1731752068406; duration=0sec 2024-11-16T10:14:28,529 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:28,529 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:A 2024-11-16T10:14:28,529 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:28,531 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:28,531 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/C is initiating minor compaction (all files) 2024-11-16T10:14:28,531 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/C in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:28,531 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/58807ce42796402ab38885b243203804, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/4784b3797cb447b4ba8038dd79dbe68c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/3b5f52bcac08446d96ca33fac531241d] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=36.7 K 2024-11-16T10:14:28,532 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58807ce42796402ab38885b243203804, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731752065296 2024-11-16T10:14:28,533 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4784b3797cb447b4ba8038dd79dbe68c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731752065958 2024-11-16T10:14:28,535 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b5f52bcac08446d96ca33fac531241d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731752067112 2024-11-16T10:14:28,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/4227477caaa541679cac71e35df17134 is 50, key is test_row_0/B:col10/1731752067180/Put/seqid=0 2024-11-16T10:14:28,546 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#C#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:28,547 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/257ffcb4b5a245c083a388fe544c5f81 is 50, key is test_row_0/C:col10/1731752067115/Put/seqid=0 2024-11-16T10:14:28,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741909_1085 (size=12301) 2024-11-16T10:14:28,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741910_1086 (size=13085) 2024-11-16T10:14:28,847 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/008a4a42e20c4934acd82ae31692e312 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/008a4a42e20c4934acd82ae31692e312 2024-11-16T10:14:28,854 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/B of 2bab7dc9c814fe637b384a46040e48b5 into 008a4a42e20c4934acd82ae31692e312(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:28,854 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:28,854 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/B, priority=13, startTime=1731752068406; duration=0sec 2024-11-16T10:14:28,854 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:28,854 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:B 2024-11-16T10:14:28,955 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/4227477caaa541679cac71e35df17134 2024-11-16T10:14:28,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/8e707e95adf84e47b2f196c481c8781f is 50, key is test_row_0/C:col10/1731752067180/Put/seqid=0 2024-11-16T10:14:28,974 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/257ffcb4b5a245c083a388fe544c5f81 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/257ffcb4b5a245c083a388fe544c5f81 2024-11-16T10:14:28,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741911_1087 (size=12301) 2024-11-16T10:14:28,976 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/8e707e95adf84e47b2f196c481c8781f 2024-11-16T10:14:28,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/cffabf1318e840e4b655bd650af85a36 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/cffabf1318e840e4b655bd650af85a36 2024-11-16T10:14:28,983 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/C of 2bab7dc9c814fe637b384a46040e48b5 into 257ffcb4b5a245c083a388fe544c5f81(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:28,983 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:28,983 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/C, priority=13, startTime=1731752068407; duration=0sec 2024-11-16T10:14:28,983 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:28,983 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:C 2024-11-16T10:14:28,988 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/cffabf1318e840e4b655bd650af85a36, entries=150, sequenceid=388, filesize=12.0 K 2024-11-16T10:14:28,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/4227477caaa541679cac71e35df17134 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4227477caaa541679cac71e35df17134 2024-11-16T10:14:28,998 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4227477caaa541679cac71e35df17134, entries=150, sequenceid=388, filesize=12.0 K 2024-11-16T10:14:29,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/8e707e95adf84e47b2f196c481c8781f as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/8e707e95adf84e47b2f196c481c8781f 2024-11-16T10:14:29,010 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/8e707e95adf84e47b2f196c481c8781f, entries=150, sequenceid=388, filesize=12.0 K 2024-11-16T10:14:29,011 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=0 B/0 for 2bab7dc9c814fe637b384a46040e48b5 in 543ms, sequenceid=388, compaction requested=false 2024-11-16T10:14:29,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:29,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:29,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-16T10:14:29,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-16T10:14:29,016 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-16T10:14:29,016 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7730 sec 2024-11-16T10:14:29,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.7820 sec 2024-11-16T10:14:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:29,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T10:14:29,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:29,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:29,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:29,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:29,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:29,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:29,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/8592641d61c848deb0776c4fb5c92526 is 50, key is test_row_0/A:col10/1731752069348/Put/seqid=0 2024-11-16T10:14:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T10:14:29,384 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:14:29,384 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:14:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-16T10:14:29,386 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:14:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T10:14:29,387 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:14:29,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:14:29,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=609, cellSize=27885, totalHeapSize=63576, min timestamp=1731752069369, max timestamp=1731752069397Num uniques 859; 2024-11-16T10:14:29,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=609, cellSize=27885, totalHeapSize=63576, min timestamp=1731752069369, max timestamp=1731752069397Num uniques 859; 2024-11-16T10:14:29,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=609, cellSize=27885, totalHeapSize=63576, min timestamp=1731752069369, max timestamp=1731752069397Num uniques 859; 2024-11-16T10:14:29,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=27885, suffix heap size=117064,new segment heap size=63576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=609 2024-11-16T10:14:29,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=27885, suffix heap size=117064,new segment heap size=63576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=609 2024-11-16T10:14:29,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=27885, suffix heap size=117064,new segment heap size=63576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=609 2024-11-16T10:14:29,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741912_1088 (size=14741) 2024-11-16T10:14:29,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/8592641d61c848deb0776c4fb5c92526 2024-11-16T10:14:29,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752129408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:29,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:29,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752129408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:29,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:29,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/6e0ba7b2220a49478d60d7eca55234cf is 50, key is test_row_0/B:col10/1731752069348/Put/seqid=0 2024-11-16T10:14:29,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:29,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752129410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:29,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:29,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:29,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752129412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:29,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:29,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752129412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:29,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:29,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:29,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741913_1089 (size=12301) 2024-11-16T10:14:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T10:14:29,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752129524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:29,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752129524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:29,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752129524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:29,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752129534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752129535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:29,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-16T10:14:29,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:29,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:29,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:29,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:29,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:29,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-16T10:14:29,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:29,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:29,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:29,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:29,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:29,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T10:14:29,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752129734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:29,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:29,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752129735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:29,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752129735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:29,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:29,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752129744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:29,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:29,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:29,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752129744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:29,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:29,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:29,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/6e0ba7b2220a49478d60d7eca55234cf 2024-11-16T10:14:29,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/b3a08876d2bd4cc8b41734f1e81de2da is 50, key is test_row_0/C:col10/1731752069348/Put/seqid=0 2024-11-16T10:14:29,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-16T10:14:29,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:29,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:29,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:29,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:29,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:29,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:29,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741914_1090 (size=12301) 2024-11-16T10:14:30,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-16T10:14:30,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:30,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:30,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:30,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:30,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:30,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:30,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T10:14:30,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:30,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752130044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:30,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752130044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:30,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752130044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:30,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:30,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:30,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:30,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752130054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752130054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:30,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-16T10:14:30,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:30,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:30,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:30,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:30,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:30,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/b3a08876d2bd4cc8b41734f1e81de2da 2024-11-16T10:14:30,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/8592641d61c848deb0776c4fb5c92526 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/8592641d61c848deb0776c4fb5c92526 2024-11-16T10:14:30,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/8592641d61c848deb0776c4fb5c92526, entries=200, sequenceid=401, filesize=14.4 K 2024-11-16T10:14:30,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/6e0ba7b2220a49478d60d7eca55234cf as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6e0ba7b2220a49478d60d7eca55234cf 2024-11-16T10:14:30,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6e0ba7b2220a49478d60d7eca55234cf, entries=150, sequenceid=401, filesize=12.0 K 2024-11-16T10:14:30,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/b3a08876d2bd4cc8b41734f1e81de2da as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/b3a08876d2bd4cc8b41734f1e81de2da 2024-11-16T10:14:30,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/b3a08876d2bd4cc8b41734f1e81de2da, entries=150, sequenceid=401, filesize=12.0 K 2024-11-16T10:14:30,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 2bab7dc9c814fe637b384a46040e48b5 in 940ms, sequenceid=401, compaction requested=true 2024-11-16T10:14:30,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:30,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:14:30,297 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:30,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:30,297 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:30,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:14:30,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:30,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:14:30,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:30,299 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:30,299 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:30,299 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/A is initiating minor compaction (all files) 2024-11-16T10:14:30,299 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/B is initiating minor compaction (all files) 2024-11-16T10:14:30,299 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/A in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:30,299 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/B in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:30,299 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/008a4a42e20c4934acd82ae31692e312, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4227477caaa541679cac71e35df17134, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6e0ba7b2220a49478d60d7eca55234cf] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=36.8 K 2024-11-16T10:14:30,299 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/ec32153741b447d9a8b9168301e84c1b, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/cffabf1318e840e4b655bd650af85a36, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/8592641d61c848deb0776c4fb5c92526] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=39.2 K 2024-11-16T10:14:30,300 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 008a4a42e20c4934acd82ae31692e312, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731752067112 2024-11-16T10:14:30,300 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec32153741b447d9a8b9168301e84c1b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731752067112 2024-11-16T10:14:30,300 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 4227477caaa541679cac71e35df17134, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731752067170 2024-11-16T10:14:30,300 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting cffabf1318e840e4b655bd650af85a36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731752067170 2024-11-16T10:14:30,300 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e0ba7b2220a49478d60d7eca55234cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731752069348 2024-11-16T10:14:30,300 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8592641d61c848deb0776c4fb5c92526, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731752069348 2024-11-16T10:14:30,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-16T10:14:30,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:30,310 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=147.60 KB heapSize=382.64 KB 2024-11-16T10:14:30,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:30,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:30,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:30,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:30,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:30,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:30,311 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#A#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:30,314 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/53afc0e15e134d21b3cf42804e43b1a7 is 50, key is test_row_0/A:col10/1731752069348/Put/seqid=0 2024-11-16T10:14:30,316 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#B#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:30,317 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/4c244380c9fc4fea871b59aabf578192 is 50, key is test_row_0/B:col10/1731752069348/Put/seqid=0 2024-11-16T10:14:30,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/a0173488e74845dd9479326e960c6735 is 50, key is test_row_0/A:col10/1731752069397/Put/seqid=0 2024-11-16T10:14:30,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741915_1091 (size=13187) 2024-11-16T10:14:30,333 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/53afc0e15e134d21b3cf42804e43b1a7 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/53afc0e15e134d21b3cf42804e43b1a7 2024-11-16T10:14:30,342 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/A of 2bab7dc9c814fe637b384a46040e48b5 into 53afc0e15e134d21b3cf42804e43b1a7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:30,342 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:30,342 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/A, priority=13, startTime=1731752070297; duration=0sec 2024-11-16T10:14:30,343 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:30,343 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:A 2024-11-16T10:14:30,343 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:30,345 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:30,345 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/C is initiating minor compaction (all files) 2024-11-16T10:14:30,345 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/C in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:30,346 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/257ffcb4b5a245c083a388fe544c5f81, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/8e707e95adf84e47b2f196c481c8781f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/b3a08876d2bd4cc8b41734f1e81de2da] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=36.8 K 2024-11-16T10:14:30,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741916_1092 (size=13187) 2024-11-16T10:14:30,346 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 257ffcb4b5a245c083a388fe544c5f81, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731752067112 2024-11-16T10:14:30,347 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e707e95adf84e47b2f196c481c8781f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731752067170 2024-11-16T10:14:30,348 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3a08876d2bd4cc8b41734f1e81de2da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731752069348 2024-11-16T10:14:30,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741917_1093 (size=12301) 2024-11-16T10:14:30,351 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/a0173488e74845dd9479326e960c6735 2024-11-16T10:14:30,356 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/4c244380c9fc4fea871b59aabf578192 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4c244380c9fc4fea871b59aabf578192 2024-11-16T10:14:30,368 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/B of 2bab7dc9c814fe637b384a46040e48b5 into 4c244380c9fc4fea871b59aabf578192(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:30,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/798c3bd73b474bc883f90d1b0869cfd6 is 50, key is test_row_0/B:col10/1731752069397/Put/seqid=0 2024-11-16T10:14:30,368 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:30,368 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/B, priority=13, startTime=1731752070297; duration=0sec 2024-11-16T10:14:30,368 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:30,368 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:B 2024-11-16T10:14:30,374 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#C#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:30,375 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/65ab35dc4ea840faa7d37634d910d070 is 50, key is test_row_0/C:col10/1731752069348/Put/seqid=0 2024-11-16T10:14:30,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741918_1094 (size=12301) 2024-11-16T10:14:30,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741919_1095 (size=13187) 2024-11-16T10:14:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T10:14:30,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:30,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:30,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:30,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752130562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:30,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:30,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752130562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:30,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:30,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752130563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752130564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752130564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:30,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752130675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:30,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752130675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752130675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:30,778 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/798c3bd73b474bc883f90d1b0869cfd6 2024-11-16T10:14:30,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/676e40ac2e7d417cb45b5b8aa68edc64 is 50, key is test_row_0/C:col10/1731752069397/Put/seqid=0 2024-11-16T10:14:30,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741920_1096 (size=12301) 2024-11-16T10:14:30,796 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/65ab35dc4ea840faa7d37634d910d070 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/65ab35dc4ea840faa7d37634d910d070 2024-11-16T10:14:30,802 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/C of 2bab7dc9c814fe637b384a46040e48b5 into 65ab35dc4ea840faa7d37634d910d070(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:30,802 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:30,802 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/C, priority=13, startTime=1731752070298; duration=0sec 2024-11-16T10:14:30,802 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:30,802 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:C 2024-11-16T10:14:30,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752130884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:30,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:30,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752130884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:30,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:30,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:30,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752130884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:30,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:30,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:31,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752131194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:31,197 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/676e40ac2e7d417cb45b5b8aa68edc64 2024-11-16T10:14:31,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:31,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752131194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:31,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:31,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752131194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:31,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:31,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:31,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/a0173488e74845dd9479326e960c6735 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/a0173488e74845dd9479326e960c6735 2024-11-16T10:14:31,208 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/a0173488e74845dd9479326e960c6735, entries=150, sequenceid=431, filesize=12.0 K 2024-11-16T10:14:31,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/798c3bd73b474bc883f90d1b0869cfd6 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/798c3bd73b474bc883f90d1b0869cfd6 2024-11-16T10:14:31,214 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/798c3bd73b474bc883f90d1b0869cfd6, entries=150, sequenceid=431, filesize=12.0 K 2024-11-16T10:14:31,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/676e40ac2e7d417cb45b5b8aa68edc64 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/676e40ac2e7d417cb45b5b8aa68edc64 2024-11-16T10:14:31,225 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/676e40ac2e7d417cb45b5b8aa68edc64, entries=150, sequenceid=431, filesize=12.0 K 2024-11-16T10:14:31,226 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~382.59 KB/391776, currentSize=46.96 KB/48090 for 2bab7dc9c814fe637b384a46040e48b5 in 916ms, sequenceid=431, compaction requested=false 2024-11-16T10:14:31,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:31,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:31,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-16T10:14:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-16T10:14:31,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-16T10:14:31,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8410 sec 2024-11-16T10:14:31,233 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.8460 sec 2024-11-16T10:14:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T10:14:31,534 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:14:31,535 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:14:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees 2024-11-16T10:14:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T10:14:31,537 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:14:31,538 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:14:31,538 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:14:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:31,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T10:14:31,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:31,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:31,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:31,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:31,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:31,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:31,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/5ea4a2839d554d2193c0ace4267ea233 is 50, key is test_row_0/A:col10/1731752070559/Put/seqid=0 2024-11-16T10:14:31,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741921_1097 (size=14741) 2024-11-16T10:14:31,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752131624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:31,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752071593, max timestamp=1731752071623Num uniques 859; 2024-11-16T10:14:31,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752071593, max timestamp=1731752071623Num uniques 859; 2024-11-16T10:14:31,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752071593, max timestamp=1731752071623Num uniques 859; 2024-11-16T10:14:31,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:14:31,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:14:31,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:14:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T10:14:31,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-16T10:14:31,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:31,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:31,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:31,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:31,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:31,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752131704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:31,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752131706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:31,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:31,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:31,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752131708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752131708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:31,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752131734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:31,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:31,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752131824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:31,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:31,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:31,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752131824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:31,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:31,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:31,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752131824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:31,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:31,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-16T10:14:31,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:31,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:31,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:31,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:31,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:31,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:31,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T10:14:31,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:31,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752131944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:31,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:31,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:31,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/5ea4a2839d554d2193c0ace4267ea233 2024-11-16T10:14:31,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-16T10:14:31,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:31,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:31,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:31,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:31,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:31,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/f7312ac949e84f9c91ec54a14d74819c is 50, key is test_row_0/B:col10/1731752070559/Put/seqid=0 2024-11-16T10:14:31,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:32,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741922_1098 (size=12301) 2024-11-16T10:14:32,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:32,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:32,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752132034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:32,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752132034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:32,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:32,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752132044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:32,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:32,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-16T10:14:32,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:32,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:32,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:32,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:32,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:32,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T10:14:32,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:32,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752132254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:32,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-16T10:14:32,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:32,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:32,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:32,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:32,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:32,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:32,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752132344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:32,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752132344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:32,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:32,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752132353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:32,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:32,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/f7312ac949e84f9c91ec54a14d74819c 2024-11-16T10:14:32,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/0a998aeb13d6449ca7ceff91709215f4 is 50, key is test_row_0/C:col10/1731752070559/Put/seqid=0 2024-11-16T10:14:32,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741923_1099 (size=12301) 2024-11-16T10:14:32,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-16T10:14:32,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:32,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:32,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:32,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:32,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:32,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:32,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-16T10:14:32,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:32,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:32,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:32,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:32,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:32,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:32,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T10:14:32,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752132723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:32,729 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_0' in 2bab7dc9c814fe637b384a46040e48b5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:14:32,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:32,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-16T10:14:32,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:32,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:32,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:32,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:32,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:32,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:32,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752132764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:32,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:32,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/0a998aeb13d6449ca7ceff91709215f4 2024-11-16T10:14:32,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/5ea4a2839d554d2193c0ace4267ea233 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/5ea4a2839d554d2193c0ace4267ea233 2024-11-16T10:14:32,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/5ea4a2839d554d2193c0ace4267ea233, entries=200, sequenceid=445, filesize=14.4 K 2024-11-16T10:14:32,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/f7312ac949e84f9c91ec54a14d74819c as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/f7312ac949e84f9c91ec54a14d74819c 2024-11-16T10:14:32,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/f7312ac949e84f9c91ec54a14d74819c, entries=150, sequenceid=445, filesize=12.0 K 2024-11-16T10:14:32,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/0a998aeb13d6449ca7ceff91709215f4 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/0a998aeb13d6449ca7ceff91709215f4 2024-11-16T10:14:32,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:32,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752132854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:32,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/0a998aeb13d6449ca7ceff91709215f4, entries=150, sequenceid=445, filesize=12.0 K 2024-11-16T10:14:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:32,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752132865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:32,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:32,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 2bab7dc9c814fe637b384a46040e48b5 in 1301ms, sequenceid=445, compaction requested=true 2024-11-16T10:14:32,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:32,878 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:32,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:14:32,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:32,879 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:32,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:14:32,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:32,879 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:32,880 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/A is initiating minor compaction (all files) 2024-11-16T10:14:32,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:32,880 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/A in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:32,885 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/53afc0e15e134d21b3cf42804e43b1a7, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/a0173488e74845dd9479326e960c6735, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/5ea4a2839d554d2193c0ace4267ea233] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=39.3 K 2024-11-16T10:14:32,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:14:32,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:32,887 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:32,887 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/B is initiating minor compaction (all files) 2024-11-16T10:14:32,887 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/B in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:32,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-16T10:14:32,888 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4c244380c9fc4fea871b59aabf578192, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/798c3bd73b474bc883f90d1b0869cfd6, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/f7312ac949e84f9c91ec54a14d74819c] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=36.9 K 2024-11-16T10:14:32,888 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53afc0e15e134d21b3cf42804e43b1a7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731752069348 2024-11-16T10:14:32,888 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c244380c9fc4fea871b59aabf578192, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731752069348 2024-11-16T10:14:32,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:32,889 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 798c3bd73b474bc883f90d1b0869cfd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731752069397 2024-11-16T10:14:32,889 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0173488e74845dd9479326e960c6735, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731752069397 2024-11-16T10:14:32,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:32,889 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ea4a2839d554d2193c0ace4267ea233, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731752070558 2024-11-16T10:14:32,889 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting f7312ac949e84f9c91ec54a14d74819c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731752070558 2024-11-16T10:14:32,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:32,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:32,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:32,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:32,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/5522ecc7d080400dadae6ba9eba27410 is 50, key is test_row_0/A:col10/1731752072876/Put/seqid=0 2024-11-16T10:14:32,913 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#B#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:32,913 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/71c3d34acc774f56bb380c1bea5f0c35 is 50, key is test_row_0/B:col10/1731752070559/Put/seqid=0 2024-11-16T10:14:32,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-16T10:14:32,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:32,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:32,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:32,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:32,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:32,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:32,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752132933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:32,938 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#A#compaction#89 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:32,939 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/0e794f7e8b734e619a3bca585f447e47 is 50, key is test_row_0/A:col10/1731752070559/Put/seqid=0 2024-11-16T10:14:32,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741924_1100 (size=14741) 2024-11-16T10:14:32,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/5522ecc7d080400dadae6ba9eba27410 2024-11-16T10:14:32,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741925_1101 (size=13289) 2024-11-16T10:14:32,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/a8ff0c86cacf438085449ed9cfab9d48 is 50, key is test_row_0/B:col10/1731752072876/Put/seqid=0 2024-11-16T10:14:32,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741926_1102 (size=13289) 2024-11-16T10:14:33,004 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/0e794f7e8b734e619a3bca585f447e47 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/0e794f7e8b734e619a3bca585f447e47 2024-11-16T10:14:33,015 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/A of 2bab7dc9c814fe637b384a46040e48b5 into 0e794f7e8b734e619a3bca585f447e47(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:33,015 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:33,016 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/A, priority=13, startTime=1731752072878; duration=0sec 2024-11-16T10:14:33,016 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:33,016 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:A 2024-11-16T10:14:33,016 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:33,023 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:33,027 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/C is initiating minor compaction (all files) 2024-11-16T10:14:33,027 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/C in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:33,027 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/65ab35dc4ea840faa7d37634d910d070, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/676e40ac2e7d417cb45b5b8aa68edc64, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/0a998aeb13d6449ca7ceff91709215f4] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=36.9 K 2024-11-16T10:14:33,030 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65ab35dc4ea840faa7d37634d910d070, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731752069348 2024-11-16T10:14:33,031 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 676e40ac2e7d417cb45b5b8aa68edc64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731752069397 2024-11-16T10:14:33,031 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a998aeb13d6449ca7ceff91709215f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731752070558 2024-11-16T10:14:33,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741927_1103 (size=12301) 2024-11-16T10:14:33,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/a8ff0c86cacf438085449ed9cfab9d48 2024-11-16T10:14:33,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:33,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752133044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:33,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:33,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:33,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:33,065 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#C#compaction#91 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:33,066 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/882acd8b070f4c7480d9ef48e4a5ed44 is 50, key is test_row_0/C:col10/1731752070559/Put/seqid=0 2024-11-16T10:14:33,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-16T10:14:33,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:33,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:33,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:33,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:33,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:33,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/0f0ea8b8e65d4d1597c16c5a702c8eff is 50, key is test_row_0/C:col10/1731752072876/Put/seqid=0 2024-11-16T10:14:33,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741928_1104 (size=13289) 2024-11-16T10:14:33,125 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/882acd8b070f4c7480d9ef48e4a5ed44 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/882acd8b070f4c7480d9ef48e4a5ed44 2024-11-16T10:14:33,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741929_1105 (size=12301) 2024-11-16T10:14:33,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/0f0ea8b8e65d4d1597c16c5a702c8eff 2024-11-16T10:14:33,136 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/C of 2bab7dc9c814fe637b384a46040e48b5 into 882acd8b070f4c7480d9ef48e4a5ed44(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:33,136 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:33,136 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/C, priority=13, startTime=1731752072879; duration=0sec 2024-11-16T10:14:33,136 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:33,136 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:C 2024-11-16T10:14:33,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/5522ecc7d080400dadae6ba9eba27410 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/5522ecc7d080400dadae6ba9eba27410 2024-11-16T10:14:33,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/5522ecc7d080400dadae6ba9eba27410, entries=200, sequenceid=482, filesize=14.4 K 2024-11-16T10:14:33,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/a8ff0c86cacf438085449ed9cfab9d48 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a8ff0c86cacf438085449ed9cfab9d48 2024-11-16T10:14:33,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a8ff0c86cacf438085449ed9cfab9d48, entries=150, sequenceid=482, filesize=12.0 K 2024-11-16T10:14:33,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/0f0ea8b8e65d4d1597c16c5a702c8eff as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/0f0ea8b8e65d4d1597c16c5a702c8eff 2024-11-16T10:14:33,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/0f0ea8b8e65d4d1597c16c5a702c8eff, entries=150, sequenceid=482, filesize=12.0 K 2024-11-16T10:14:33,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 2bab7dc9c814fe637b384a46040e48b5 in 283ms, sequenceid=482, compaction requested=false 2024-11-16T10:14:33,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:33,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-16T10:14:33,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:33,228 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-16T10:14:33,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:33,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:33,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:33,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:33,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:33,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:33,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/f8d8ad5183554e6e8679933c61f6cc2b is 50, key is test_row_0/A:col10/1731752072909/Put/seqid=0 2024-11-16T10:14:33,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741930_1106 (size=12301) 2024-11-16T10:14:33,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:33,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:33,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752073273, max timestamp=1731752073321Num uniques 859; 2024-11-16T10:14:33,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752073273, max timestamp=1731752073321Num uniques 859; 2024-11-16T10:14:33,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752073273, max timestamp=1731752073321Num uniques 859; 2024-11-16T10:14:33,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:14:33,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:14:33,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:14:33,387 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/71c3d34acc774f56bb380c1bea5f0c35 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/71c3d34acc774f56bb380c1bea5f0c35 2024-11-16T10:14:33,395 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/B of 2bab7dc9c814fe637b384a46040e48b5 into 71c3d34acc774f56bb380c1bea5f0c35(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:33,395 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:33,395 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/B, priority=13, startTime=1731752072879; duration=0sec 2024-11-16T10:14:33,395 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:33,395 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:B 2024-11-16T10:14:33,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:33,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752133415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:33,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:33,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:33,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:33,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:33,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752133524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:33,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:33,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:33,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:33,637 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/f8d8ad5183554e6e8679933c61f6cc2b 2024-11-16T10:14:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T10:14:33,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/487294c4d4884beeaf02dc1e26ba735f is 50, key is test_row_0/B:col10/1731752072909/Put/seqid=0 2024-11-16T10:14:33,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741931_1107 (size=12301) 2024-11-16T10:14:33,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752133734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:33,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:33,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752133794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:33,794 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_0' in 2bab7dc9c814fe637b384a46040e48b5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:14:33,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:33,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:33,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:33,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752133884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:33,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752133884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:33,888 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in 2bab7dc9c814fe637b384a46040e48b5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:14:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:33,888 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in 2bab7dc9c814fe637b384a46040e48b5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:14:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:34,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752134044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:34,112 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/487294c4d4884beeaf02dc1e26ba735f 2024-11-16T10:14:34,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/3defcddc47d64ee08fedd1f94d7acb0a is 50, key is test_row_0/C:col10/1731752072909/Put/seqid=0 2024-11-16T10:14:34,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741932_1108 (size=12301) 2024-11-16T10:14:34,130 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/3defcddc47d64ee08fedd1f94d7acb0a 2024-11-16T10:14:34,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/f8d8ad5183554e6e8679933c61f6cc2b as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/f8d8ad5183554e6e8679933c61f6cc2b 2024-11-16T10:14:34,150 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/f8d8ad5183554e6e8679933c61f6cc2b, entries=150, sequenceid=493, filesize=12.0 K 2024-11-16T10:14:34,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/487294c4d4884beeaf02dc1e26ba735f as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/487294c4d4884beeaf02dc1e26ba735f 2024-11-16T10:14:34,156 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/487294c4d4884beeaf02dc1e26ba735f, entries=150, sequenceid=493, filesize=12.0 K 2024-11-16T10:14:34,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/3defcddc47d64ee08fedd1f94d7acb0a as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/3defcddc47d64ee08fedd1f94d7acb0a 2024-11-16T10:14:34,167 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/3defcddc47d64ee08fedd1f94d7acb0a, entries=150, sequenceid=493, filesize=12.0 K 2024-11-16T10:14:34,168 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 2bab7dc9c814fe637b384a46040e48b5 in 940ms, sequenceid=493, compaction requested=true 2024-11-16T10:14:34,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:34,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:34,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-11-16T10:14:34,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-11-16T10:14:34,173 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-16T10:14:34,173 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6320 sec 2024-11-16T10:14:34,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees in 2.6380 sec 2024-11-16T10:14:34,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-16T10:14:34,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:34,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T10:14:34,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:34,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T10:14:34,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:34,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T10:14:34,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:34,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/1e3482bce4b14b8dbe7aa79caa2f1c30 is 50, key is test_row_0/A:col10/1731752073398/Put/seqid=0 2024-11-16T10:14:34,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741933_1109 (size=14741) 2024-11-16T10:14:34,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:34,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752134586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:34,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:34,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:34,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752134694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:34,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:34,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:34,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:34,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43730 deadline: 1731752134754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:34,758 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_0' in 2bab7dc9c814fe637b384a46040e48b5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:14:34,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:34,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:34,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:34,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752134904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:34,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:34,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:34,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:34,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/1e3482bce4b14b8dbe7aa79caa2f1c30 2024-11-16T10:14:34,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/39f0d6bbd59646139855a47a05e20c01 is 50, key is test_row_0/B:col10/1731752073398/Put/seqid=0 2024-11-16T10:14:34,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741934_1110 (size=12301) 2024-11-16T10:14:35,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752135214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:35,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/39f0d6bbd59646139855a47a05e20c01 2024-11-16T10:14:35,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/8486cb744c6e48cea653ece29a7cfbd4 is 50, key is test_row_0/C:col10/1731752073398/Put/seqid=0 2024-11-16T10:14:35,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741935_1111 (size=12301) 2024-11-16T10:14:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T10:14:35,704 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:14:35,704 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:14:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees 2024-11-16T10:14:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-16T10:14:35,706 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=31, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:14:35,707 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=31, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:14:35,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:14:35,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752135724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:35,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/8486cb744c6e48cea653ece29a7cfbd4 2024-11-16T10:14:35,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/1e3482bce4b14b8dbe7aa79caa2f1c30 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/1e3482bce4b14b8dbe7aa79caa2f1c30 2024-11-16T10:14:35,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/1e3482bce4b14b8dbe7aa79caa2f1c30, entries=200, sequenceid=532, filesize=14.4 K 2024-11-16T10:14:35,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/39f0d6bbd59646139855a47a05e20c01 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/39f0d6bbd59646139855a47a05e20c01 2024-11-16T10:14:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-16T10:14:35,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/39f0d6bbd59646139855a47a05e20c01, entries=150, sequenceid=532, filesize=12.0 K 2024-11-16T10:14:35,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/8486cb744c6e48cea653ece29a7cfbd4 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/8486cb744c6e48cea653ece29a7cfbd4 2024-11-16T10:14:35,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/8486cb744c6e48cea653ece29a7cfbd4, entries=150, sequenceid=532, filesize=12.0 K 2024-11-16T10:14:35,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 2bab7dc9c814fe637b384a46040e48b5 in 1265ms, sequenceid=532, compaction requested=true 2024-11-16T10:14:35,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:35,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:14:35,828 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:14:35,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:35,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:14:35,828 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:14:35,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:35,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bab7dc9c814fe637b384a46040e48b5:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:14:35,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:35,829 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55072 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:14:35,829 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:14:35,829 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/A is initiating minor compaction (all files) 2024-11-16T10:14:35,829 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/B is initiating minor compaction (all files) 2024-11-16T10:14:35,830 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/A in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:35,830 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/B in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:35,830 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/0e794f7e8b734e619a3bca585f447e47, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/5522ecc7d080400dadae6ba9eba27410, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/f8d8ad5183554e6e8679933c61f6cc2b, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/1e3482bce4b14b8dbe7aa79caa2f1c30] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=53.8 K 2024-11-16T10:14:35,830 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/71c3d34acc774f56bb380c1bea5f0c35, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a8ff0c86cacf438085449ed9cfab9d48, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/487294c4d4884beeaf02dc1e26ba735f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/39f0d6bbd59646139855a47a05e20c01] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=49.0 K 2024-11-16T10:14:35,830 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 71c3d34acc774f56bb380c1bea5f0c35, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731752070558 2024-11-16T10:14:35,830 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e794f7e8b734e619a3bca585f447e47, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731752070558 2024-11-16T10:14:35,831 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting a8ff0c86cacf438085449ed9cfab9d48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1731752071704 2024-11-16T10:14:35,831 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5522ecc7d080400dadae6ba9eba27410, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1731752071704 2024-11-16T10:14:35,831 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 487294c4d4884beeaf02dc1e26ba735f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731752072901 2024-11-16T10:14:35,831 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8d8ad5183554e6e8679933c61f6cc2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731752072901 2024-11-16T10:14:35,831 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 39f0d6bbd59646139855a47a05e20c01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1731752073398 2024-11-16T10:14:35,831 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e3482bce4b14b8dbe7aa79caa2f1c30, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1731752073387 2024-11-16T10:14:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:35,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T10:14:35,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:35,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:35,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:35,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:35,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:35,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:35,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/32704d44404645c8934b4c054ed0285c is 50, key is test_row_0/A:col10/1731752075827/Put/seqid=0 2024-11-16T10:14:35,842 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#A#compaction#100 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:35,842 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/db009d0a4ccc429197fea3318884173a is 50, key is test_row_0/A:col10/1731752073398/Put/seqid=0 2024-11-16T10:14:35,845 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#B#compaction#101 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:35,846 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/4bd3ed849aa047dc93ca498df234d6d4 is 50, key is test_row_0/B:col10/1731752073398/Put/seqid=0 2024-11-16T10:14:35,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741937_1113 (size=13425) 2024-11-16T10:14:35,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-16T10:14:35,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:35,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:35,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:35,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:35,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:35,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741936_1112 (size=14741) 2024-11-16T10:14:35,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741938_1114 (size=13425) 2024-11-16T10:14:35,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731752075871, max timestamp=1731752075916Num uniques 859; 2024-11-16T10:14:35,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731752075871, max timestamp=1731752075916Num uniques 859; 2024-11-16T10:14:35,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731752075871, max timestamp=1731752075916Num uniques 859; 2024-11-16T10:14:35,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T10:14:35,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T10:14:35,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T10:14:35,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752135945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:35,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:35,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:35,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:35,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752135953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:35,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752135954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:35,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:35,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:35,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:35,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:35,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:35,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:36,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-16T10:14:36,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:36,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:36,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:36,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:36,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:36,023 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-16T10:14:36,023 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:14:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-16T10:14:36,027 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:36,028 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-16T10:14:36,028 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:14:36,028 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:36,028 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:36,028 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:36,028 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:14:36,028 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:14:36,030 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-16T10:14:36,030 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:14:36,030 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:36,030 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:36,030 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:14:36,037 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-16T10:14:36,037 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:14:36,037 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:36,037 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:36,037 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:14:36,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:36,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752136065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:36,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752136065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752136065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:36,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:36,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:36,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:36,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:36,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:36,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:36,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:36,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:36,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:36,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-16T10:14:36,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:36,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:36,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:36,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:36,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:36,265 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/db009d0a4ccc429197fea3318884173a as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/db009d0a4ccc429197fea3318884173a 2024-11-16T10:14:36,273 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/A of 2bab7dc9c814fe637b384a46040e48b5 into db009d0a4ccc429197fea3318884173a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:36,273 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:36,273 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/A, priority=12, startTime=1731752075827; duration=0sec 2024-11-16T10:14:36,273 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:36,273 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:A 2024-11-16T10:14:36,273 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:14:36,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:36,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:36,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752136274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752136274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752136275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:36,275 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:14:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:36,275 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 2bab7dc9c814fe637b384a46040e48b5/C is initiating minor compaction (all files) 2024-11-16T10:14:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:36,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:36,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:36,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:36,276 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bab7dc9c814fe637b384a46040e48b5/C in TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:36,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:36,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:36,276 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/882acd8b070f4c7480d9ef48e4a5ed44, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/0f0ea8b8e65d4d1597c16c5a702c8eff, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/3defcddc47d64ee08fedd1f94d7acb0a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/8486cb744c6e48cea653ece29a7cfbd4] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp, totalSize=49.0 K 2024-11-16T10:14:36,276 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 882acd8b070f4c7480d9ef48e4a5ed44, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731752070558 2024-11-16T10:14:36,277 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f0ea8b8e65d4d1597c16c5a702c8eff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1731752071704 2024-11-16T10:14:36,277 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3defcddc47d64ee08fedd1f94d7acb0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731752072901 2024-11-16T10:14:36,278 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8486cb744c6e48cea653ece29a7cfbd4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1731752073398 2024-11-16T10:14:36,286 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bab7dc9c814fe637b384a46040e48b5#C#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:36,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/32704d44404645c8934b4c054ed0285c 2024-11-16T10:14:36,287 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/727b29f5fe744275a399740f5c3127b6 is 50, key is test_row_0/C:col10/1731752073398/Put/seqid=0 2024-11-16T10:14:36,293 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/4bd3ed849aa047dc93ca498df234d6d4 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4bd3ed849aa047dc93ca498df234d6d4 2024-11-16T10:14:36,298 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/B of 2bab7dc9c814fe637b384a46040e48b5 into 4bd3ed849aa047dc93ca498df234d6d4(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:36,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/77b9fb53e7bb4c92aa25025fd7077fe9 is 50, key is test_row_0/B:col10/1731752075827/Put/seqid=0 2024-11-16T10:14:36,298 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:36,298 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/B, priority=12, startTime=1731752075828; duration=0sec 2024-11-16T10:14:36,298 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:36,298 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:B 2024-11-16T10:14:36,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741939_1115 (size=13425) 2024-11-16T10:14:36,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741940_1116 (size=12301) 2024-11-16T10:14:36,306 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/727b29f5fe744275a399740f5c3127b6 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/727b29f5fe744275a399740f5c3127b6 2024-11-16T10:14:36,311 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2bab7dc9c814fe637b384a46040e48b5/C of 2bab7dc9c814fe637b384a46040e48b5 into 727b29f5fe744275a399740f5c3127b6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:36,311 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:36,311 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., storeName=2bab7dc9c814fe637b384a46040e48b5/C, priority=12, startTime=1731752075828; duration=0sec 2024-11-16T10:14:36,312 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:36,312 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bab7dc9c814fe637b384a46040e48b5:C 2024-11-16T10:14:36,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-16T10:14:36,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:36,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:36,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:36,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:36,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:36,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-16T10:14:36,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-16T10:14:36,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:36,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:36,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:36,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:36,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:36,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:36,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:36,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752136585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752136586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752136586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:36,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:36,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-16T10:14:36,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:36,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:36,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:36,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:36,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:36,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/77b9fb53e7bb4c92aa25025fd7077fe9 2024-11-16T10:14:36,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/f89c2a033b754085823d3db0746eb0cd is 50, key is test_row_0/C:col10/1731752075827/Put/seqid=0 2024-11-16T10:14:36,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741941_1117 (size=12301) 2024-11-16T10:14:36,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43746 deadline: 1731752136734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:36,735 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in 2bab7dc9c814fe637b384a46040e48b5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:14:36,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:36,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:36,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:36,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-16T10:14:36,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:36,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:36,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:36,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:36,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:36,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-16T10:14:36,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-16T10:14:36,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:36,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:36,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:36,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:36,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:36,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:37,027 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T10:14:37,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:37,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43692 deadline: 1731752137095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1731752137095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:37,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:37,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:37,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:37,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:37,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:37,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:37,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-16T10:14:37,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:37,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:37,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:37,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:37,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:37,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:37,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43706 deadline: 1731752137105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:14:37,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/f89c2a033b754085823d3db0746eb0cd 2024-11-16T10:14:37,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/32704d44404645c8934b4c054ed0285c as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/32704d44404645c8934b4c054ed0285c 2024-11-16T10:14:37,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/32704d44404645c8934b4c054ed0285c, entries=200, sequenceid=542, filesize=14.4 K 2024-11-16T10:14:37,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/77b9fb53e7bb4c92aa25025fd7077fe9 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/77b9fb53e7bb4c92aa25025fd7077fe9 2024-11-16T10:14:37,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/77b9fb53e7bb4c92aa25025fd7077fe9, entries=150, sequenceid=542, filesize=12.0 K 2024-11-16T10:14:37,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/f89c2a033b754085823d3db0746eb0cd as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/f89c2a033b754085823d3db0746eb0cd 2024-11-16T10:14:37,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/f89c2a033b754085823d3db0746eb0cd, entries=150, sequenceid=542, filesize=12.0 K 2024-11-16T10:14:37,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 2bab7dc9c814fe637b384a46040e48b5 in 1305ms, sequenceid=542, compaction requested=false 2024-11-16T10:14:37,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:37,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-16T10:14:37,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:37,256 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-16T10:14:37,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:37,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:37,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:37,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:37,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:37,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:37,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/97a24bcd2ece47278b3421b702c7a7ae is 50, key is test_row_0/A:col10/1731752075940/Put/seqid=0 2024-11-16T10:14:37,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741942_1118 (size=12301) 2024-11-16T10:14:37,666 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/97a24bcd2ece47278b3421b702c7a7ae 2024-11-16T10:14:37,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/6c248c65a5424a528772bbf36b19d5d8 is 50, key is test_row_0/B:col10/1731752075940/Put/seqid=0 2024-11-16T10:14:37,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741943_1119 (size=12301) 2024-11-16T10:14:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-16T10:14:38,087 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/6c248c65a5424a528772bbf36b19d5d8 2024-11-16T10:14:38,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/9cc8343fa66145298da83383730d13d3 is 50, key is test_row_0/C:col10/1731752075940/Put/seqid=0 2024-11-16T10:14:38,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741944_1120 (size=12301) 2024-11-16T10:14:38,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:38,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. as already flushing 2024-11-16T10:14:38,106 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-16T10:14:38,106 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-16T10:14:38,106 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:14:38,106 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:14:38,106 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:38,106 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:38,106 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:38,106 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:38,106 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:14:38,106 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:14:38,125 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-16T10:14:38,125 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:14:38,125 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:38,125 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:38,125 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:14:38,504 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/9cc8343fa66145298da83383730d13d3 2024-11-16T10:14:38,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/97a24bcd2ece47278b3421b702c7a7ae as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/97a24bcd2ece47278b3421b702c7a7ae 2024-11-16T10:14:38,519 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/97a24bcd2ece47278b3421b702c7a7ae, entries=150, sequenceid=580, filesize=12.0 K 2024-11-16T10:14:38,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/6c248c65a5424a528772bbf36b19d5d8 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6c248c65a5424a528772bbf36b19d5d8 2024-11-16T10:14:38,525 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6c248c65a5424a528772bbf36b19d5d8, entries=150, sequenceid=580, filesize=12.0 K 2024-11-16T10:14:38,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/9cc8343fa66145298da83383730d13d3 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/9cc8343fa66145298da83383730d13d3 2024-11-16T10:14:38,531 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/9cc8343fa66145298da83383730d13d3, entries=150, sequenceid=580, filesize=12.0 K 2024-11-16T10:14:38,532 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=20.13 KB/20610 for 2bab7dc9c814fe637b384a46040e48b5 in 1275ms, sequenceid=580, compaction requested=true 2024-11-16T10:14:38,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2603): Flush status journal for 2bab7dc9c814fe637b384a46040e48b5: 2024-11-16T10:14:38,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:38,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=32 2024-11-16T10:14:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=32 2024-11-16T10:14:38,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-16T10:14:38,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8260 sec 2024-11-16T10:14:38,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees in 2.8310 sec 2024-11-16T10:14:38,746 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-16T10:14:38,746 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:14:38,746 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:38,746 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:38,747 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:14:38,789 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-16T10:14:38,789 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:14:38,789 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:38,789 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:38,790 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:14:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-16T10:14:39,865 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:14:39,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T10:14:39,866 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-16T10:14:39,866 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-16T10:14:39,866 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-16T10:14:39,866 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-16T10:14:39,867 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-11-16T10:14:39,867 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T10:14:39,867 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3002 2024-11-16T10:14:39,867 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2735 2024-11-16T10:14:39,867 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T10:14:39,868 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1266 2024-11-16T10:14:39,868 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3797 rows 2024-11-16T10:14:39,868 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1270 2024-11-16T10:14:39,868 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3806 rows 2024-11-16T10:14:39,868 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T10:14:39,868 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T10:14:39,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:39,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:39,869 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:14:39,872 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T10:14:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T10:14:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-16T10:14:39,878 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752079878"}]},"ts":"1731752079878"} 2024-11-16T10:14:39,880 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T10:14:39,880 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T10:14:39,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T10:14:39,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bab7dc9c814fe637b384a46040e48b5, UNASSIGN}] 2024-11-16T10:14:39,886 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bab7dc9c814fe637b384a46040e48b5, UNASSIGN 2024-11-16T10:14:39,887 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=2bab7dc9c814fe637b384a46040e48b5, regionState=CLOSING, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:39,889 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bab7dc9c814fe637b384a46040e48b5, UNASSIGN because future has completed 2024-11-16T10:14:39,890 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T10:14:39,890 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912}] 2024-11-16T10:14:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-16T10:14:40,047 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:40,047 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T10:14:40,048 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 2bab7dc9c814fe637b384a46040e48b5, disabling compactions & flushes 2024-11-16T10:14:40,048 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:40,048 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:40,048 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. after waiting 0 ms 2024-11-16T10:14:40,048 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:40,048 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(2902): Flushing 2bab7dc9c814fe637b384a46040e48b5 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-16T10:14:40,049 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=A 2024-11-16T10:14:40,049 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:40,049 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=B 2024-11-16T10:14:40,049 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:40,049 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bab7dc9c814fe637b384a46040e48b5, store=C 2024-11-16T10:14:40,049 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:40,055 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/e97d7b0c92a445218400ee7ab5c4dcc0 is 50, key is test_row_0/A:col10/1731752078786/Put/seqid=0 2024-11-16T10:14:40,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741945_1121 (size=12301) 2024-11-16T10:14:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-16T10:14:40,459 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/e97d7b0c92a445218400ee7ab5c4dcc0 2024-11-16T10:14:40,467 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/3ee1fdd102f1485e9f159f5dc71dc1cd is 50, key is test_row_0/B:col10/1731752078786/Put/seqid=0 2024-11-16T10:14:40,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741946_1122 (size=12301) 2024-11-16T10:14:40,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-16T10:14:40,871 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/3ee1fdd102f1485e9f159f5dc71dc1cd 2024-11-16T10:14:40,879 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/c18a95e8cfdc452e89023937ea043a77 is 50, key is test_row_0/C:col10/1731752078786/Put/seqid=0 2024-11-16T10:14:40,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741947_1123 (size=12301) 2024-11-16T10:14:41,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-16T10:14:41,283 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/c18a95e8cfdc452e89023937ea043a77 2024-11-16T10:14:41,289 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/A/e97d7b0c92a445218400ee7ab5c4dcc0 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/e97d7b0c92a445218400ee7ab5c4dcc0 2024-11-16T10:14:41,295 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/e97d7b0c92a445218400ee7ab5c4dcc0, entries=150, sequenceid=588, filesize=12.0 K 2024-11-16T10:14:41,297 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/B/3ee1fdd102f1485e9f159f5dc71dc1cd as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/3ee1fdd102f1485e9f159f5dc71dc1cd 2024-11-16T10:14:41,302 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/3ee1fdd102f1485e9f159f5dc71dc1cd, entries=150, sequenceid=588, filesize=12.0 K 2024-11-16T10:14:41,305 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/.tmp/C/c18a95e8cfdc452e89023937ea043a77 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/c18a95e8cfdc452e89023937ea043a77 2024-11-16T10:14:41,311 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/c18a95e8cfdc452e89023937ea043a77, entries=150, sequenceid=588, filesize=12.0 K 2024-11-16T10:14:41,312 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 2bab7dc9c814fe637b384a46040e48b5 in 1264ms, sequenceid=588, compaction requested=true 2024-11-16T10:14:41,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/4a6ee23f28ab4e28b1732549b25bf823, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/03c54c9fc3f94b0eb16a177c60a81d6d, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/1a0dc691bc8f41efb7ab04a2e509a60c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/fe096d1e5553454298c10a7c04eb5f41, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/0dff2173001444cfb847c2c7f6f90e2e, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/d49c3bb5f9674e36a000a56dd47f6ccd, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/dd8c6aa16a5e418e9cfbc6587d333ee1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/40c8cb5845a04993b6c9d14eb8e8c207, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/1115ad2bb2534544b8661c0169c6ff31, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/c65f5bb7856d427596a33aa19a006ecc, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/01a1fd11b6fc4916b16dbc41c11462df, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/7302ca6703474608a88f4611cb73619c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/44721fb754644a618b73ea89980e32ef, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/d68479c495034b459b3b68c1fe809561, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/64db3a91f8cf4f7883d63e414b35cc96, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/10e40e471fd24da887d6c07c515ae15c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/c10438f825cc46ce8b9fc54dd453e5f0, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/33da2c5562f3462d9d10af4aadc0e975, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/2351409015d942299168b37daf81fbe7, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/4b716fb13c8a4241b89ab037a62142ae, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/ecc9da2ff0cb4caa99ad81d82e277eab, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/81d8f57231344d7e9d843a62a7298435, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/531ce653a2224b40a0c43beae26ba4ef, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/ec32153741b447d9a8b9168301e84c1b, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/cffabf1318e840e4b655bd650af85a36, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/8592641d61c848deb0776c4fb5c92526, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/53afc0e15e134d21b3cf42804e43b1a7, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/a0173488e74845dd9479326e960c6735, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/5ea4a2839d554d2193c0ace4267ea233, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/0e794f7e8b734e619a3bca585f447e47, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/5522ecc7d080400dadae6ba9eba27410, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/f8d8ad5183554e6e8679933c61f6cc2b, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/1e3482bce4b14b8dbe7aa79caa2f1c30] to archive 2024-11-16T10:14:41,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:14:41,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/4a6ee23f28ab4e28b1732549b25bf823 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/4a6ee23f28ab4e28b1732549b25bf823 2024-11-16T10:14:41,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/03c54c9fc3f94b0eb16a177c60a81d6d to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/03c54c9fc3f94b0eb16a177c60a81d6d 2024-11-16T10:14:41,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/1a0dc691bc8f41efb7ab04a2e509a60c to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/1a0dc691bc8f41efb7ab04a2e509a60c 2024-11-16T10:14:41,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/fe096d1e5553454298c10a7c04eb5f41 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/fe096d1e5553454298c10a7c04eb5f41 2024-11-16T10:14:41,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/0dff2173001444cfb847c2c7f6f90e2e to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/0dff2173001444cfb847c2c7f6f90e2e 2024-11-16T10:14:41,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/d49c3bb5f9674e36a000a56dd47f6ccd to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/d49c3bb5f9674e36a000a56dd47f6ccd 2024-11-16T10:14:41,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/dd8c6aa16a5e418e9cfbc6587d333ee1 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/dd8c6aa16a5e418e9cfbc6587d333ee1 2024-11-16T10:14:41,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/40c8cb5845a04993b6c9d14eb8e8c207 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/40c8cb5845a04993b6c9d14eb8e8c207 2024-11-16T10:14:41,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/1115ad2bb2534544b8661c0169c6ff31 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/1115ad2bb2534544b8661c0169c6ff31 2024-11-16T10:14:41,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/c65f5bb7856d427596a33aa19a006ecc to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/c65f5bb7856d427596a33aa19a006ecc 2024-11-16T10:14:41,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/01a1fd11b6fc4916b16dbc41c11462df to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/01a1fd11b6fc4916b16dbc41c11462df 2024-11-16T10:14:41,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/7302ca6703474608a88f4611cb73619c to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/7302ca6703474608a88f4611cb73619c 2024-11-16T10:14:41,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/44721fb754644a618b73ea89980e32ef to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/44721fb754644a618b73ea89980e32ef 2024-11-16T10:14:41,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/d68479c495034b459b3b68c1fe809561 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/d68479c495034b459b3b68c1fe809561 2024-11-16T10:14:41,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/64db3a91f8cf4f7883d63e414b35cc96 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/64db3a91f8cf4f7883d63e414b35cc96 2024-11-16T10:14:41,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/10e40e471fd24da887d6c07c515ae15c to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/10e40e471fd24da887d6c07c515ae15c 2024-11-16T10:14:41,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/c10438f825cc46ce8b9fc54dd453e5f0 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/c10438f825cc46ce8b9fc54dd453e5f0 2024-11-16T10:14:41,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/33da2c5562f3462d9d10af4aadc0e975 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/33da2c5562f3462d9d10af4aadc0e975 2024-11-16T10:14:41,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/2351409015d942299168b37daf81fbe7 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/2351409015d942299168b37daf81fbe7 2024-11-16T10:14:41,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/4b716fb13c8a4241b89ab037a62142ae to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/4b716fb13c8a4241b89ab037a62142ae 2024-11-16T10:14:41,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/ecc9da2ff0cb4caa99ad81d82e277eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/ecc9da2ff0cb4caa99ad81d82e277eab 2024-11-16T10:14:41,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/81d8f57231344d7e9d843a62a7298435 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/81d8f57231344d7e9d843a62a7298435 2024-11-16T10:14:41,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/531ce653a2224b40a0c43beae26ba4ef to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/531ce653a2224b40a0c43beae26ba4ef 2024-11-16T10:14:41,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/ec32153741b447d9a8b9168301e84c1b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/ec32153741b447d9a8b9168301e84c1b 2024-11-16T10:14:41,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/cffabf1318e840e4b655bd650af85a36 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/cffabf1318e840e4b655bd650af85a36 2024-11-16T10:14:41,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/8592641d61c848deb0776c4fb5c92526 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/8592641d61c848deb0776c4fb5c92526 2024-11-16T10:14:41,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/53afc0e15e134d21b3cf42804e43b1a7 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/53afc0e15e134d21b3cf42804e43b1a7 2024-11-16T10:14:41,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/a0173488e74845dd9479326e960c6735 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/a0173488e74845dd9479326e960c6735 2024-11-16T10:14:41,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/5ea4a2839d554d2193c0ace4267ea233 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/5ea4a2839d554d2193c0ace4267ea233 2024-11-16T10:14:41,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/0e794f7e8b734e619a3bca585f447e47 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/0e794f7e8b734e619a3bca585f447e47 2024-11-16T10:14:41,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/5522ecc7d080400dadae6ba9eba27410 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/5522ecc7d080400dadae6ba9eba27410 2024-11-16T10:14:41,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/f8d8ad5183554e6e8679933c61f6cc2b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/f8d8ad5183554e6e8679933c61f6cc2b 2024-11-16T10:14:41,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/1e3482bce4b14b8dbe7aa79caa2f1c30 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/1e3482bce4b14b8dbe7aa79caa2f1c30 2024-11-16T10:14:41,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/1913d37f8ca640e9a7b1ae6b69b3bc92, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/1e835da9a95d4eeab64270f9d41e6cf6, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/09e1e8bce8e748c6b5db0a89e26233ba, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a9c5966d6e524b088aefcbe441ceeafb, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/13917a7b449648bbb94ac3e3fd509289, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a0ffbc669a594e7da3b5104a3e3eaebe, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/2663a08d1c454300a159faa8af6ceaf2, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6c86d9862fc84da99d664698cfa75a02, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/9b353d14d376437eb1e8fbf4ae64d39c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/101f8f8236654296bbec0ad06686af7c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/c7576978ee234523a8abe3ef6e278c66, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4c8772ff729c4bc7a8cfae3f4f7e18b5, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a8c5607f98ae43899e762d2091e5267c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4897668181194e71951b92497f27f7de, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/9f3549a2a57a43e6aeb58a4a18a7ba12, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6ef08f67c7704da4afbdb13c8515d5b4, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6e5a9f56285b4cad9109d530674382fb, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/322e3908a6cc4d22b992e7e2ac1976ec, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/ac5b24185f0744abb9f8c97d5dc83a55, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/61ec9b01c0ef4a25a9201f7a5bdf8b15, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/9f3dfab7b31046ebbe5596855dbbf5b6, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/17c127d8914b4198886e621e99afc575, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/008a4a42e20c4934acd82ae31692e312, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/5b1aaeb0ed3c49f0b5bca761b65519ff, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4227477caaa541679cac71e35df17134, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4c244380c9fc4fea871b59aabf578192, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6e0ba7b2220a49478d60d7eca55234cf, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/798c3bd73b474bc883f90d1b0869cfd6, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/71c3d34acc774f56bb380c1bea5f0c35, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/f7312ac949e84f9c91ec54a14d74819c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a8ff0c86cacf438085449ed9cfab9d48, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/487294c4d4884beeaf02dc1e26ba735f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/39f0d6bbd59646139855a47a05e20c01] to archive 2024-11-16T10:14:41,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:14:41,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/1913d37f8ca640e9a7b1ae6b69b3bc92 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/1913d37f8ca640e9a7b1ae6b69b3bc92 2024-11-16T10:14:41,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/1e835da9a95d4eeab64270f9d41e6cf6 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/1e835da9a95d4eeab64270f9d41e6cf6 2024-11-16T10:14:41,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/09e1e8bce8e748c6b5db0a89e26233ba to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/09e1e8bce8e748c6b5db0a89e26233ba 2024-11-16T10:14:41,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a9c5966d6e524b088aefcbe441ceeafb to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a9c5966d6e524b088aefcbe441ceeafb 2024-11-16T10:14:41,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/13917a7b449648bbb94ac3e3fd509289 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/13917a7b449648bbb94ac3e3fd509289 2024-11-16T10:14:41,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a0ffbc669a594e7da3b5104a3e3eaebe to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a0ffbc669a594e7da3b5104a3e3eaebe 2024-11-16T10:14:41,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/2663a08d1c454300a159faa8af6ceaf2 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/2663a08d1c454300a159faa8af6ceaf2 2024-11-16T10:14:41,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6c86d9862fc84da99d664698cfa75a02 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6c86d9862fc84da99d664698cfa75a02 2024-11-16T10:14:41,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/9b353d14d376437eb1e8fbf4ae64d39c to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/9b353d14d376437eb1e8fbf4ae64d39c 2024-11-16T10:14:41,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/101f8f8236654296bbec0ad06686af7c to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/101f8f8236654296bbec0ad06686af7c 2024-11-16T10:14:41,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/c7576978ee234523a8abe3ef6e278c66 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/c7576978ee234523a8abe3ef6e278c66 2024-11-16T10:14:41,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4c8772ff729c4bc7a8cfae3f4f7e18b5 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4c8772ff729c4bc7a8cfae3f4f7e18b5 2024-11-16T10:14:41,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a8c5607f98ae43899e762d2091e5267c to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a8c5607f98ae43899e762d2091e5267c 2024-11-16T10:14:41,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4897668181194e71951b92497f27f7de to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4897668181194e71951b92497f27f7de 2024-11-16T10:14:41,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/9f3549a2a57a43e6aeb58a4a18a7ba12 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/9f3549a2a57a43e6aeb58a4a18a7ba12 2024-11-16T10:14:41,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6ef08f67c7704da4afbdb13c8515d5b4 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6ef08f67c7704da4afbdb13c8515d5b4 2024-11-16T10:14:41,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6e5a9f56285b4cad9109d530674382fb to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6e5a9f56285b4cad9109d530674382fb 2024-11-16T10:14:41,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/322e3908a6cc4d22b992e7e2ac1976ec to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/322e3908a6cc4d22b992e7e2ac1976ec 2024-11-16T10:14:41,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/ac5b24185f0744abb9f8c97d5dc83a55 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/ac5b24185f0744abb9f8c97d5dc83a55 2024-11-16T10:14:41,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/61ec9b01c0ef4a25a9201f7a5bdf8b15 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/61ec9b01c0ef4a25a9201f7a5bdf8b15 2024-11-16T10:14:41,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/9f3dfab7b31046ebbe5596855dbbf5b6 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/9f3dfab7b31046ebbe5596855dbbf5b6 2024-11-16T10:14:41,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/17c127d8914b4198886e621e99afc575 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/17c127d8914b4198886e621e99afc575 2024-11-16T10:14:41,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/008a4a42e20c4934acd82ae31692e312 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/008a4a42e20c4934acd82ae31692e312 2024-11-16T10:14:41,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/5b1aaeb0ed3c49f0b5bca761b65519ff to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/5b1aaeb0ed3c49f0b5bca761b65519ff 2024-11-16T10:14:41,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4227477caaa541679cac71e35df17134 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4227477caaa541679cac71e35df17134 2024-11-16T10:14:41,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4c244380c9fc4fea871b59aabf578192 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4c244380c9fc4fea871b59aabf578192 2024-11-16T10:14:41,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6e0ba7b2220a49478d60d7eca55234cf to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6e0ba7b2220a49478d60d7eca55234cf 2024-11-16T10:14:41,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/798c3bd73b474bc883f90d1b0869cfd6 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/798c3bd73b474bc883f90d1b0869cfd6 2024-11-16T10:14:41,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/71c3d34acc774f56bb380c1bea5f0c35 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/71c3d34acc774f56bb380c1bea5f0c35 2024-11-16T10:14:41,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/f7312ac949e84f9c91ec54a14d74819c to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/f7312ac949e84f9c91ec54a14d74819c 2024-11-16T10:14:41,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a8ff0c86cacf438085449ed9cfab9d48 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/a8ff0c86cacf438085449ed9cfab9d48 2024-11-16T10:14:41,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/487294c4d4884beeaf02dc1e26ba735f to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/487294c4d4884beeaf02dc1e26ba735f 2024-11-16T10:14:41,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/39f0d6bbd59646139855a47a05e20c01 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/39f0d6bbd59646139855a47a05e20c01 2024-11-16T10:14:41,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/04f6e1d9288944e0a5b94e1239655795, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/8576be72d18544f5930f9edd974a5021, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/84559caf512843578976691f461ffa0a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/4bc59a12e99d46f4bf3678b4d5956c6b, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/2897ef71927e4086b771847973b9b33f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/2e46bd55c2cd48faa4e9b178527dfa82, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/cd1878fea611407fb24c0d373ebb2977, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/b88225d89ccf4691aac95f989247a132, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/6596d32419ea4269afbccbf8db8dd914, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/c8a029bb9d6346d3a3ebb1b6ed0da479, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/cf9aaa7a34ac4371822303d9480217e9, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/b3c421255ed947d0b65f7951d5d141ff, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/1bc90786d8714b09bc8e1d88ac9b03d3, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/d2c03aab8e4d43ebbfa23c8ccfc30198, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/29d6cfc16f75456ca64f074079442fab, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/a9110092f80e4c0b9aa1afbed8a0853a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/1a13375fe32e4019aca5847eef2f0f03, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/f6402589a8874ee8b000e8628aa141ee, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/13b25096a7934a6f8246c51b4b44f190, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/58807ce42796402ab38885b243203804, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/fafef004ca4e43528c432cc760c2558e, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/4784b3797cb447b4ba8038dd79dbe68c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/257ffcb4b5a245c083a388fe544c5f81, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/3b5f52bcac08446d96ca33fac531241d, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/8e707e95adf84e47b2f196c481c8781f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/65ab35dc4ea840faa7d37634d910d070, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/b3a08876d2bd4cc8b41734f1e81de2da, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/676e40ac2e7d417cb45b5b8aa68edc64, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/882acd8b070f4c7480d9ef48e4a5ed44, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/0a998aeb13d6449ca7ceff91709215f4, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/0f0ea8b8e65d4d1597c16c5a702c8eff, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/3defcddc47d64ee08fedd1f94d7acb0a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/8486cb744c6e48cea653ece29a7cfbd4] to archive 2024-11-16T10:14:41,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:14:41,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/04f6e1d9288944e0a5b94e1239655795 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/04f6e1d9288944e0a5b94e1239655795 2024-11-16T10:14:41,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/8576be72d18544f5930f9edd974a5021 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/8576be72d18544f5930f9edd974a5021 2024-11-16T10:14:41,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/84559caf512843578976691f461ffa0a to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/84559caf512843578976691f461ffa0a 2024-11-16T10:14:41,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/4bc59a12e99d46f4bf3678b4d5956c6b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/4bc59a12e99d46f4bf3678b4d5956c6b 2024-11-16T10:14:41,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/2897ef71927e4086b771847973b9b33f to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/2897ef71927e4086b771847973b9b33f 2024-11-16T10:14:41,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/2e46bd55c2cd48faa4e9b178527dfa82 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/2e46bd55c2cd48faa4e9b178527dfa82 2024-11-16T10:14:41,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/cd1878fea611407fb24c0d373ebb2977 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/cd1878fea611407fb24c0d373ebb2977 2024-11-16T10:14:41,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/b88225d89ccf4691aac95f989247a132 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/b88225d89ccf4691aac95f989247a132 2024-11-16T10:14:41,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/6596d32419ea4269afbccbf8db8dd914 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/6596d32419ea4269afbccbf8db8dd914 2024-11-16T10:14:41,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/c8a029bb9d6346d3a3ebb1b6ed0da479 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/c8a029bb9d6346d3a3ebb1b6ed0da479 2024-11-16T10:14:41,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/cf9aaa7a34ac4371822303d9480217e9 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/cf9aaa7a34ac4371822303d9480217e9 2024-11-16T10:14:41,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/b3c421255ed947d0b65f7951d5d141ff to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/b3c421255ed947d0b65f7951d5d141ff 2024-11-16T10:14:41,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/1bc90786d8714b09bc8e1d88ac9b03d3 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/1bc90786d8714b09bc8e1d88ac9b03d3 2024-11-16T10:14:41,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/d2c03aab8e4d43ebbfa23c8ccfc30198 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/d2c03aab8e4d43ebbfa23c8ccfc30198 2024-11-16T10:14:41,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/29d6cfc16f75456ca64f074079442fab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/29d6cfc16f75456ca64f074079442fab 2024-11-16T10:14:41,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/a9110092f80e4c0b9aa1afbed8a0853a to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/a9110092f80e4c0b9aa1afbed8a0853a 2024-11-16T10:14:41,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/1a13375fe32e4019aca5847eef2f0f03 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/1a13375fe32e4019aca5847eef2f0f03 2024-11-16T10:14:41,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/f6402589a8874ee8b000e8628aa141ee to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/f6402589a8874ee8b000e8628aa141ee 2024-11-16T10:14:41,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/13b25096a7934a6f8246c51b4b44f190 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/13b25096a7934a6f8246c51b4b44f190 2024-11-16T10:14:41,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/58807ce42796402ab38885b243203804 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/58807ce42796402ab38885b243203804 2024-11-16T10:14:41,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/fafef004ca4e43528c432cc760c2558e to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/fafef004ca4e43528c432cc760c2558e 2024-11-16T10:14:41,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/4784b3797cb447b4ba8038dd79dbe68c to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/4784b3797cb447b4ba8038dd79dbe68c 2024-11-16T10:14:41,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/257ffcb4b5a245c083a388fe544c5f81 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/257ffcb4b5a245c083a388fe544c5f81 2024-11-16T10:14:41,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/3b5f52bcac08446d96ca33fac531241d to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/3b5f52bcac08446d96ca33fac531241d 2024-11-16T10:14:41,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/8e707e95adf84e47b2f196c481c8781f to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/8e707e95adf84e47b2f196c481c8781f 2024-11-16T10:14:41,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/65ab35dc4ea840faa7d37634d910d070 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/65ab35dc4ea840faa7d37634d910d070 2024-11-16T10:14:41,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/b3a08876d2bd4cc8b41734f1e81de2da to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/b3a08876d2bd4cc8b41734f1e81de2da 2024-11-16T10:14:41,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/676e40ac2e7d417cb45b5b8aa68edc64 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/676e40ac2e7d417cb45b5b8aa68edc64 2024-11-16T10:14:41,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/882acd8b070f4c7480d9ef48e4a5ed44 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/882acd8b070f4c7480d9ef48e4a5ed44 2024-11-16T10:14:41,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/0a998aeb13d6449ca7ceff91709215f4 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/0a998aeb13d6449ca7ceff91709215f4 2024-11-16T10:14:41,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/0f0ea8b8e65d4d1597c16c5a702c8eff to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/0f0ea8b8e65d4d1597c16c5a702c8eff 2024-11-16T10:14:41,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/3defcddc47d64ee08fedd1f94d7acb0a to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/3defcddc47d64ee08fedd1f94d7acb0a 2024-11-16T10:14:41,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/8486cb744c6e48cea653ece29a7cfbd4 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/8486cb744c6e48cea653ece29a7cfbd4 2024-11-16T10:14:41,537 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/recovered.edits/591.seqid, newMaxSeqId=591, maxSeqId=1 2024-11-16T10:14:41,539 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. 2024-11-16T10:14:41,539 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 2bab7dc9c814fe637b384a46040e48b5: Waiting for close lock at 1731752080048Running coprocessor pre-close hooks at 1731752080048Disabling compacts and flushes for region at 1731752080048Disabling writes for close at 1731752080048Obtaining lock to block concurrent updates at 1731752080048Preparing flush snapshotting stores in 2bab7dc9c814fe637b384a46040e48b5 at 1731752080048Finished memstore snapshotting TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731752080049 (+1 ms)Flushing stores of TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5. at 1731752080050 (+1 ms)Flushing 2bab7dc9c814fe637b384a46040e48b5/A: creating writer at 1731752080050Flushing 2bab7dc9c814fe637b384a46040e48b5/A: appending metadata at 1731752080054 (+4 ms)Flushing 2bab7dc9c814fe637b384a46040e48b5/A: closing flushed file at 1731752080054Flushing 2bab7dc9c814fe637b384a46040e48b5/B: creating writer at 1731752080464 (+410 ms)Flushing 2bab7dc9c814fe637b384a46040e48b5/B: appending metadata at 1731752080466 (+2 ms)Flushing 2bab7dc9c814fe637b384a46040e48b5/B: closing flushed file at 1731752080466Flushing 2bab7dc9c814fe637b384a46040e48b5/C: creating writer at 1731752080875 (+409 ms)Flushing 2bab7dc9c814fe637b384a46040e48b5/C: appending metadata at 1731752080878 (+3 ms)Flushing 2bab7dc9c814fe637b384a46040e48b5/C: closing flushed file at 1731752080878Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72efe680: reopening flushed file at 1731752081288 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fe564e6: reopening flushed file at 1731752081295 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f91ab61: reopening flushed file at 1731752081303 (+8 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 2bab7dc9c814fe637b384a46040e48b5 in 1264ms, sequenceid=588, compaction requested=true at 1731752081312 (+9 ms)Writing region close event to WAL at 1731752081533 (+221 ms)Running coprocessor post-close hooks at 1731752081539 (+6 ms)Closed at 1731752081539 2024-11-16T10:14:41,542 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:41,542 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=2bab7dc9c814fe637b384a46040e48b5, regionState=CLOSED 2024-11-16T10:14:41,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 because future has completed 2024-11-16T10:14:41,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-16T10:14:41,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 2bab7dc9c814fe637b384a46040e48b5, server=b16ff38458c1,45417,1731752048912 in 1.6550 sec 2024-11-16T10:14:41,549 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-16T10:14:41,549 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bab7dc9c814fe637b384a46040e48b5, UNASSIGN in 1.6620 sec 2024-11-16T10:14:41,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-16T10:14:41,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6690 sec 2024-11-16T10:14:41,552 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752081552"}]},"ts":"1731752081552"} 2024-11-16T10:14:41,554 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T10:14:41,554 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T10:14:41,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6810 sec 2024-11-16T10:14:42,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-16T10:14:42,025 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:14:42,029 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T10:14:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:14:42,039 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=37, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:14:42,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-16T10:14:42,041 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=37, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:14:42,045 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:42,049 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A, FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B, FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C, FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/recovered.edits] 2024-11-16T10:14:42,052 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/32704d44404645c8934b4c054ed0285c to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/32704d44404645c8934b4c054ed0285c 2024-11-16T10:14:42,054 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/97a24bcd2ece47278b3421b702c7a7ae to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/97a24bcd2ece47278b3421b702c7a7ae 2024-11-16T10:14:42,055 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/db009d0a4ccc429197fea3318884173a to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/db009d0a4ccc429197fea3318884173a 2024-11-16T10:14:42,057 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/e97d7b0c92a445218400ee7ab5c4dcc0 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/A/e97d7b0c92a445218400ee7ab5c4dcc0 2024-11-16T10:14:42,059 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/3ee1fdd102f1485e9f159f5dc71dc1cd to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/3ee1fdd102f1485e9f159f5dc71dc1cd 2024-11-16T10:14:42,061 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4bd3ed849aa047dc93ca498df234d6d4 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/4bd3ed849aa047dc93ca498df234d6d4 2024-11-16T10:14:42,062 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6c248c65a5424a528772bbf36b19d5d8 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/6c248c65a5424a528772bbf36b19d5d8 2024-11-16T10:14:42,063 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/77b9fb53e7bb4c92aa25025fd7077fe9 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/B/77b9fb53e7bb4c92aa25025fd7077fe9 2024-11-16T10:14:42,066 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/727b29f5fe744275a399740f5c3127b6 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/727b29f5fe744275a399740f5c3127b6 2024-11-16T10:14:42,067 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/9cc8343fa66145298da83383730d13d3 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/9cc8343fa66145298da83383730d13d3 2024-11-16T10:14:42,068 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/c18a95e8cfdc452e89023937ea043a77 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/c18a95e8cfdc452e89023937ea043a77 2024-11-16T10:14:42,070 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/f89c2a033b754085823d3db0746eb0cd to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/C/f89c2a033b754085823d3db0746eb0cd 2024-11-16T10:14:42,073 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/recovered.edits/591.seqid to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5/recovered.edits/591.seqid 2024-11-16T10:14:42,073 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/2bab7dc9c814fe637b384a46040e48b5 2024-11-16T10:14:42,073 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T10:14:42,078 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=37, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:14:42,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-16T10:14:42,087 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T10:14:42,118 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T10:14:42,119 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=37, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:14:42,120 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T10:14:42,120 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731752082120"}]},"ts":"9223372036854775807"} 2024-11-16T10:14:42,124 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T10:14:42,124 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2bab7dc9c814fe637b384a46040e48b5, NAME => 'TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T10:14:42,124 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T10:14:42,124 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731752082124"}]},"ts":"9223372036854775807"} 2024-11-16T10:14:42,127 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T10:14:42,128 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=37, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:14:42,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 97 msec 2024-11-16T10:14:42,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-16T10:14:42,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T10:14:42,145 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:14:42,160 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=249 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-607888602_22 at /127.0.0.1:59204 [Waiting for operation #95] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;b16ff38458c1:45417-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-607888602_22 at /127.0.0.1:41782 [Waiting for operation #300] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1174335200_22 at /127.0.0.1:39150 [Waiting for operation #286] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=314 (was 281) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3882 (was 4499) 2024-11-16T10:14:42,169 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=314, ProcessCount=11, AvailableMemoryMB=3881 2024-11-16T10:14:42,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T10:14:42,171 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T10:14:42,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T10:14:42,173 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T10:14:42,173 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:42,173 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 38 2024-11-16T10:14:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T10:14:42,175 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T10:14:42,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741948_1124 (size=1061) 2024-11-16T10:14:42,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T10:14:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T10:14:42,584 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0 2024-11-16T10:14:42,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741949_1125 (size=53) 2024-11-16T10:14:42,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T10:14:42,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:14:42,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8cc2406be4637cf7e35e71b99fcd3d28, disabling compactions & flushes 2024-11-16T10:14:42,995 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:42,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:42,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. after waiting 0 ms 2024-11-16T10:14:42,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:42,995 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:42,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8cc2406be4637cf7e35e71b99fcd3d28: Waiting for close lock at 1731752082995Disabling compacts and flushes for region at 1731752082995Disabling writes for close at 1731752082995Writing region close event to WAL at 1731752082995Closed at 1731752082995 2024-11-16T10:14:42,996 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T10:14:42,997 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731752082996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731752082996"}]},"ts":"1731752082996"} 2024-11-16T10:14:42,999 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T10:14:43,000 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T10:14:43,001 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752083000"}]},"ts":"1731752083000"} 2024-11-16T10:14:43,003 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T10:14:43,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8cc2406be4637cf7e35e71b99fcd3d28, ASSIGN}] 2024-11-16T10:14:43,005 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8cc2406be4637cf7e35e71b99fcd3d28, ASSIGN 2024-11-16T10:14:43,007 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8cc2406be4637cf7e35e71b99fcd3d28, ASSIGN; state=OFFLINE, location=b16ff38458c1,45417,1731752048912; forceNewPlan=false, retain=false 2024-11-16T10:14:43,158 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=8cc2406be4637cf7e35e71b99fcd3d28, regionState=OPENING, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:43,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8cc2406be4637cf7e35e71b99fcd3d28, ASSIGN because future has completed 2024-11-16T10:14:43,165 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912}] 2024-11-16T10:14:43,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T10:14:43,325 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:43,326 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:14:43,326 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:43,326 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:14:43,326 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:43,326 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:43,327 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:43,329 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:14:43,330 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8cc2406be4637cf7e35e71b99fcd3d28 columnFamilyName A 2024-11-16T10:14:43,330 DEBUG [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:43,330 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] regionserver.HStore(327): Store=8cc2406be4637cf7e35e71b99fcd3d28/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:14:43,330 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:43,332 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:14:43,332 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8cc2406be4637cf7e35e71b99fcd3d28 columnFamilyName B 2024-11-16T10:14:43,332 DEBUG [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:43,332 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] regionserver.HStore(327): Store=8cc2406be4637cf7e35e71b99fcd3d28/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:14:43,332 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:43,334 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:14:43,334 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8cc2406be4637cf7e35e71b99fcd3d28 columnFamilyName C 2024-11-16T10:14:43,334 DEBUG [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:43,334 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] regionserver.HStore(327): Store=8cc2406be4637cf7e35e71b99fcd3d28/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:14:43,334 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:43,335 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:43,335 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:43,336 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:43,337 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:43,337 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:43,338 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:14:43,339 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:43,341 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T10:14:43,342 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 8cc2406be4637cf7e35e71b99fcd3d28; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62936745, jitterRate=-0.06216941773891449}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:14:43,342 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:43,343 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 8cc2406be4637cf7e35e71b99fcd3d28: Running coprocessor pre-open hook at 1731752083326Writing region info on filesystem at 1731752083326Initializing all the Stores at 1731752083327 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752083327Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752083327Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752083327Cleaning up temporary data from old regions at 1731752083337 (+10 ms)Running coprocessor post-open hooks at 1731752083342 (+5 ms)Region opened successfully at 1731752083343 (+1 ms) 2024-11-16T10:14:43,343 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., pid=40, masterSystemTime=1731752083318 2024-11-16T10:14:43,345 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:43,345 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:43,346 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=8cc2406be4637cf7e35e71b99fcd3d28, regionState=OPEN, openSeqNum=2, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:43,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 because future has completed 2024-11-16T10:14:43,351 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-16T10:14:43,351 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 in 184 msec 2024-11-16T10:14:43,352 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-16T10:14:43,352 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8cc2406be4637cf7e35e71b99fcd3d28, ASSIGN in 348 msec 2024-11-16T10:14:43,353 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T10:14:43,353 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752083353"}]},"ts":"1731752083353"} 2024-11-16T10:14:43,354 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T10:14:43,355 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T10:14:43,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1840 sec 2024-11-16T10:14:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T10:14:44,326 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:14:44,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:44,329 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:14:44,330 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:14:44,332 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:14:44,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:14:44,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:14:44,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12b44f32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:44,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:14:44,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:14:44,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:44,335 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:14:44,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b24555, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:44,336 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:14:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:14:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:14:44,339 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:14:44,341 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T10:14:44,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is b16ff38458c1,45949,1731752047888 2024-11-16T10:14:44,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1bf6b1e5 2024-11-16T10:14:44,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T10:14:44,343 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T10:14:44,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T10:14:44,348 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T10:14:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T10:14:44,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741950_1126 (size=1097) 2024-11-16T10:14:44,767 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-16T10:14:44,768 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-16T10:14:44,774 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T10:14:44,782 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8cc2406be4637cf7e35e71b99fcd3d28, REOPEN/MOVE}] 2024-11-16T10:14:44,783 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8cc2406be4637cf7e35e71b99fcd3d28, REOPEN/MOVE 2024-11-16T10:14:44,784 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=43 updating hbase:meta row=8cc2406be4637cf7e35e71b99fcd3d28, regionState=CLOSING, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:44,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8cc2406be4637cf7e35e71b99fcd3d28, REOPEN/MOVE because future has completed 2024-11-16T10:14:44,786 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T10:14:44,786 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912}] 2024-11-16T10:14:44,939 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] handler.UnassignRegionHandler(122): Close 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:44,939 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T10:14:44,939 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1722): Closing 8cc2406be4637cf7e35e71b99fcd3d28, disabling compactions & flushes 2024-11-16T10:14:44,939 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:44,939 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:44,939 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. after waiting 0 ms 2024-11-16T10:14:44,939 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:44,943 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T10:14:44,944 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:44,944 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1676): Region close journal for 8cc2406be4637cf7e35e71b99fcd3d28: Waiting for close lock at 1731752084939Running coprocessor pre-close hooks at 1731752084939Disabling compacts and flushes for region at 1731752084939Disabling writes for close at 1731752084939Writing region close event to WAL at 1731752084940 (+1 ms)Running coprocessor post-close hooks at 1731752084944 (+4 ms)Closed at 1731752084944 2024-11-16T10:14:44,944 WARN [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegionServer(3299): Not adding moved region record: 8cc2406be4637cf7e35e71b99fcd3d28 to self. 2024-11-16T10:14:44,946 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] handler.UnassignRegionHandler(157): Closed 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:44,947 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=43 updating hbase:meta row=8cc2406be4637cf7e35e71b99fcd3d28, regionState=CLOSED 2024-11-16T10:14:44,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 because future has completed 2024-11-16T10:14:44,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-11-16T10:14:44,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; CloseRegionProcedure 8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 in 164 msec 2024-11-16T10:14:44,952 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8cc2406be4637cf7e35e71b99fcd3d28, REOPEN/MOVE; state=CLOSED, location=b16ff38458c1,45417,1731752048912; forceNewPlan=false, retain=true 2024-11-16T10:14:45,103 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=43 updating hbase:meta row=8cc2406be4637cf7e35e71b99fcd3d28, regionState=OPENING, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:45,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8cc2406be4637cf7e35e71b99fcd3d28, REOPEN/MOVE because future has completed 2024-11-16T10:14:45,107 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=43, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912}] 2024-11-16T10:14:45,264 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:45,264 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:14:45,264 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:45,264 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:14:45,264 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:45,264 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:45,269 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:45,270 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:14:45,274 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8cc2406be4637cf7e35e71b99fcd3d28 columnFamilyName A 2024-11-16T10:14:45,276 DEBUG [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:45,276 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] regionserver.HStore(327): Store=8cc2406be4637cf7e35e71b99fcd3d28/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:14:45,277 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:45,278 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:14:45,278 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8cc2406be4637cf7e35e71b99fcd3d28 columnFamilyName B 2024-11-16T10:14:45,278 DEBUG [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:45,278 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] regionserver.HStore(327): Store=8cc2406be4637cf7e35e71b99fcd3d28/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:14:45,279 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:45,279 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:14:45,279 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8cc2406be4637cf7e35e71b99fcd3d28 columnFamilyName C 2024-11-16T10:14:45,279 DEBUG [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:45,280 INFO [StoreOpener-8cc2406be4637cf7e35e71b99fcd3d28-1 {}] regionserver.HStore(327): Store=8cc2406be4637cf7e35e71b99fcd3d28/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:14:45,280 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:45,280 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:45,281 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:45,282 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:45,283 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:45,283 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:45,283 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:14:45,285 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:45,286 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 8cc2406be4637cf7e35e71b99fcd3d28; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71046719, jitterRate=0.05867861211299896}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:14:45,286 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:45,287 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 8cc2406be4637cf7e35e71b99fcd3d28: Running coprocessor pre-open hook at 1731752085265Writing region info on filesystem at 1731752085265Initializing all the Stores at 1731752085266 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752085266Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752085266Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752085266Cleaning up temporary data from old regions at 1731752085283 (+17 ms)Running coprocessor post-open hooks at 1731752085286 (+3 ms)Region opened successfully at 1731752085287 (+1 ms) 2024-11-16T10:14:45,287 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., pid=45, masterSystemTime=1731752085260 2024-11-16T10:14:45,289 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:45,289 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:45,290 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=43 updating hbase:meta row=8cc2406be4637cf7e35e71b99fcd3d28, regionState=OPEN, openSeqNum=5, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:45,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=43, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 because future has completed 2024-11-16T10:14:45,294 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=43 2024-11-16T10:14:45,294 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=43, state=SUCCESS, hasLock=false; OpenRegionProcedure 8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 in 185 msec 2024-11-16T10:14:45,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-16T10:14:45,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8cc2406be4637cf7e35e71b99fcd3d28, REOPEN/MOVE in 512 msec 2024-11-16T10:14:45,299 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=41 2024-11-16T10:14:45,299 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=41, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 524 msec 2024-11-16T10:14:45,302 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 950 msec 2024-11-16T10:14:45,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=41 2024-11-16T10:14:45,304 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:14:45,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:14:45,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:14:45,306 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:14:45,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:14:45,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:14:45,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73cccc71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:14:45,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:14:45,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:45,308 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:14:45,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@289e5f6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,309 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,309 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:14:45,310 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:14:45,310 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:14:45,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:14:45,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:14:45,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@441bf5fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:14:45,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:14:45,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:45,312 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:14:45,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692706e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,314 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:14:45,314 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:14:45,315 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:14:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:14:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:14:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cd4475, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:14:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:14:45,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:45,316 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:14:45,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@329ed148, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,318 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:14:45,318 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:14:45,319 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:14:45,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:14:45,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:14:45,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f2cf1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:14:45,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:14:45,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:45,321 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:14:45,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6072724d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,323 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:14:45,323 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:14:45,324 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:14:45,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:14:45,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:14:45,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b9c011, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:14:45,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:14:45,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:45,325 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:14:45,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a640a9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,327 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,327 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:14:45,327 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:14:45,328 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:14:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:14:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:14:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f7582b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:14:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:14:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:45,329 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:14:45,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15bf2458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,331 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:14:45,331 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:14:45,332 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:14:45,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:14:45,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:14:45,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2559f6b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:14:45,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:14:45,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:45,333 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:14:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43e409a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,334 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:14:45,334 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:14:45,335 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:14:45,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:14:45,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:14:45,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0ba3a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:14:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:14:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:45,336 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:14:45,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39e0e3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,337 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:14:45,338 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:14:45,338 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:14:45,338 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:14:45,338 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:14:45,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2941dea5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:14:45,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:14:45,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:14:45,340 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:14:45,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@219244cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:14:45,341 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:14:45,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:14:45,342 DEBUG [Thread-641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:14:45,342 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:14:45,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:14:45,343 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:14:45,343 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:14:45,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:14:45,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:14:45,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:14:45,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:14:45,345 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:14:45,347 DEBUG [Thread-644 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:14:45,347 DEBUG [Thread-633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:14:45,347 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:14:45,348 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:14:45,348 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:14:45,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:14:45,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:14:45,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:14:45,349 DEBUG [Thread-648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:14:45,349 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:14:45,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:14:45,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:14:45,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:14:45,350 DEBUG [Thread-648 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:14:45,351 DEBUG [Thread-646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:14:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:14:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:14:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-16T10:14:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:14:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:14:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:14:45,352 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:14:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:14:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:14:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T10:14:45,353 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:14:45,353 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:14:45,353 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:14:45,354 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:14:45,354 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:14:45,354 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:14:45,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:14:45,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:14:45,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:14:45,357 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:14:45,357 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:14:45,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:14:45,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:14:45,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:14:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:45,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8cc2406be4637cf7e35e71b99fcd3d28 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T10:14:45,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=A 2024-11-16T10:14:45,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:45,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=B 2024-11-16T10:14:45,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:45,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=C 2024-11-16T10:14:45,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:45,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168784dbcd7b344652886c55c7c1fee759_8cc2406be4637cf7e35e71b99fcd3d28 is 50, key is test_row_0/A:col10/1731752085386/Put/seqid=0 2024-11-16T10:14:45,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:45,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752145437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752145438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:45,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752145438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:45,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752145446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:45,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752145446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:45,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T10:14:45,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741951_1127 (size=12154) 2024-11-16T10:14:45,468 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:45,474 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168784dbcd7b344652886c55c7c1fee759_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168784dbcd7b344652886c55c7c1fee759_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:45,475 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/517fb434304e492db3a596197c1da6b5, store: [table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:45,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/517fb434304e492db3a596197c1da6b5 is 175, key is test_row_0/A:col10/1731752085386/Put/seqid=0 2024-11-16T10:14:45,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741952_1128 (size=30955) 2024-11-16T10:14:45,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T10:14:45,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:45,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:45,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:45,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:45,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:45,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752145556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:45,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752145556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:45,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752145556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:45,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:45,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752145565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:45,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:45,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752145565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:45,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:45,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:45,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T10:14:45,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:45,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:45,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:45,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:45,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T10:14:45,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752145764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:45,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752145764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:45,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752145764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:45,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752145774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:45,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:45,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752145774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:45,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:45,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:45,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T10:14:45,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:45,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:45,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:45,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:45,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:45,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:45,906 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/517fb434304e492db3a596197c1da6b5 2024-11-16T10:14:45,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/87a2d1b0e1584e398819ce5f67b3d3cb is 50, key is test_row_0/B:col10/1731752085386/Put/seqid=0 2024-11-16T10:14:45,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741953_1129 (size=12001) 2024-11-16T10:14:45,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/87a2d1b0e1584e398819ce5f67b3d3cb 2024-11-16T10:14:45,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T10:14:45,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:45,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:45,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:45,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:45,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:45,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T10:14:46,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/baaf9b3ca37647e4b976997abadb0201 is 50, key is test_row_0/C:col10/1731752085386/Put/seqid=0 2024-11-16T10:14:46,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741954_1130 (size=12001) 2024-11-16T10:14:46,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/baaf9b3ca37647e4b976997abadb0201 2024-11-16T10:14:46,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/517fb434304e492db3a596197c1da6b5 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/517fb434304e492db3a596197c1da6b5 2024-11-16T10:14:46,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/517fb434304e492db3a596197c1da6b5, entries=150, sequenceid=16, filesize=30.2 K 2024-11-16T10:14:46,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/87a2d1b0e1584e398819ce5f67b3d3cb as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/87a2d1b0e1584e398819ce5f67b3d3cb 2024-11-16T10:14:46,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/87a2d1b0e1584e398819ce5f67b3d3cb, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T10:14:46,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/baaf9b3ca37647e4b976997abadb0201 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/baaf9b3ca37647e4b976997abadb0201 2024-11-16T10:14:46,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/baaf9b3ca37647e4b976997abadb0201, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T10:14:46,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8cc2406be4637cf7e35e71b99fcd3d28 in 663ms, sequenceid=16, compaction requested=false 2024-11-16T10:14:46,063 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-16T10:14:46,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:46,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752085434, max timestamp=1731752086076Num uniques 859; 2024-11-16T10:14:46,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752085434, max timestamp=1731752086076Num uniques 859; 2024-11-16T10:14:46,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:14:46,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:14:46,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:46,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8cc2406be4637cf7e35e71b99fcd3d28 3/3 column families, dataSize=64.85 KB heapSize=184.09 KB 2024-11-16T10:14:46,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=A 2024-11-16T10:14:46,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:46,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=B 2024-11-16T10:14:46,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:46,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=C 2024-11-16T10:14:46,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:46,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116493f5b3e553d4900836f3dfce857fedc_8cc2406be4637cf7e35e71b99fcd3d28 is 50, key is test_row_0/A:col10/1731752086078/Put/seqid=0 2024-11-16T10:14:46,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741955_1131 (size=12154) 2024-11-16T10:14:46,105 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,111 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116493f5b3e553d4900836f3dfce857fedc_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116493f5b3e553d4900836f3dfce857fedc_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:46,113 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/b27a0365162c4a8da9d8c3b5af7c3996, store: [table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:46,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/b27a0365162c4a8da9d8c3b5af7c3996 is 175, key is test_row_0/A:col10/1731752086078/Put/seqid=0 2024-11-16T10:14:46,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752146118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752146120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752146120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752146122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741956_1132 (size=30955) 2024-11-16T10:14:46,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T10:14:46,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:46,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:46,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:46,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:46,135 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=38, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/b27a0365162c4a8da9d8c3b5af7c3996 2024-11-16T10:14:46,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:46,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752146127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:46,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/cd23874b66a3441fad6808d08d3fad42 is 50, key is test_row_0/B:col10/1731752086078/Put/seqid=0 2024-11-16T10:14:46,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741957_1133 (size=12001) 2024-11-16T10:14:46,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/cd23874b66a3441fad6808d08d3fad42 2024-11-16T10:14:46,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/3ed2d7f08efa4efc9771fafa04522103 is 50, key is test_row_0/C:col10/1731752086078/Put/seqid=0 2024-11-16T10:14:46,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741958_1134 (size=12001) 2024-11-16T10:14:46,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/3ed2d7f08efa4efc9771fafa04522103 2024-11-16T10:14:46,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/b27a0365162c4a8da9d8c3b5af7c3996 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/b27a0365162c4a8da9d8c3b5af7c3996 2024-11-16T10:14:46,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/b27a0365162c4a8da9d8c3b5af7c3996, entries=150, sequenceid=38, filesize=30.2 K 2024-11-16T10:14:46,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/cd23874b66a3441fad6808d08d3fad42 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/cd23874b66a3441fad6808d08d3fad42 2024-11-16T10:14:46,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/cd23874b66a3441fad6808d08d3fad42, entries=150, sequenceid=38, filesize=11.7 K 2024-11-16T10:14:46,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752146234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/3ed2d7f08efa4efc9771fafa04522103 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/3ed2d7f08efa4efc9771fafa04522103 2024-11-16T10:14:46,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752146234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752146234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752146234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/3ed2d7f08efa4efc9771fafa04522103, entries=150, sequenceid=38, filesize=11.7 K 2024-11-16T10:14:46,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=114.05 KB/116790 for 8cc2406be4637cf7e35e71b99fcd3d28 in 164ms, sequenceid=38, compaction requested=false 2024-11-16T10:14:46,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:46,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8cc2406be4637cf7e35e71b99fcd3d28 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-16T10:14:46,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=A 2024-11-16T10:14:46,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:46,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=B 2024-11-16T10:14:46,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:46,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=C 2024-11-16T10:14:46,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:46,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169912af47b8ed418ca9dc9d7b2f4dc7c3_8cc2406be4637cf7e35e71b99fcd3d28 is 50, key is test_row_0/A:col10/1731752086117/Put/seqid=0 2024-11-16T10:14:46,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741959_1135 (size=14594) 2024-11-16T10:14:46,283 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T10:14:46,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:46,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:46,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:46,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:46,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:46,296 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169912af47b8ed418ca9dc9d7b2f4dc7c3_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169912af47b8ed418ca9dc9d7b2f4dc7c3_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:46,297 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/7ff504bf75cb4b7998ec6a64ca426292, store: [table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:46,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/7ff504bf75cb4b7998ec6a64ca426292 is 175, key is test_row_0/A:col10/1731752086117/Put/seqid=0 2024-11-16T10:14:46,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752146303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741960_1136 (size=39549) 2024-11-16T10:14:46,319 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=59, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/7ff504bf75cb4b7998ec6a64ca426292 2024-11-16T10:14:46,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/4c471a589d68434eb8615ea21ca72829 is 50, key is test_row_0/B:col10/1731752086117/Put/seqid=0 2024-11-16T10:14:46,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741961_1137 (size=12001) 2024-11-16T10:14:46,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/4c471a589d68434eb8615ea21ca72829 2024-11-16T10:14:46,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/7a39fb247ef54b1492e2f6cc096e0669 is 50, key is test_row_0/C:col10/1731752086117/Put/seqid=0 2024-11-16T10:14:46,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741962_1138 (size=12001) 2024-11-16T10:14:46,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/7a39fb247ef54b1492e2f6cc096e0669 2024-11-16T10:14:46,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/7ff504bf75cb4b7998ec6a64ca426292 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/7ff504bf75cb4b7998ec6a64ca426292 2024-11-16T10:14:46,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/7ff504bf75cb4b7998ec6a64ca426292, entries=200, sequenceid=59, filesize=38.6 K 2024-11-16T10:14:46,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/4c471a589d68434eb8615ea21ca72829 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/4c471a589d68434eb8615ea21ca72829 2024-11-16T10:14:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/4c471a589d68434eb8615ea21ca72829, entries=150, sequenceid=59, filesize=11.7 K 2024-11-16T10:14:46,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/7a39fb247ef54b1492e2f6cc096e0669 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/7a39fb247ef54b1492e2f6cc096e0669 2024-11-16T10:14:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/7a39fb247ef54b1492e2f6cc096e0669, entries=150, sequenceid=59, filesize=11.7 K 2024-11-16T10:14:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 8cc2406be4637cf7e35e71b99fcd3d28 in 154ms, sequenceid=59, compaction requested=true 2024-11-16T10:14:46,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:46,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8cc2406be4637cf7e35e71b99fcd3d28:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:14:46,404 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:46,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:46,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8cc2406be4637cf7e35e71b99fcd3d28:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:14:46,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:46,404 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:46,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8cc2406be4637cf7e35e71b99fcd3d28:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:14:46,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,405 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:46,405 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:46,405 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 8cc2406be4637cf7e35e71b99fcd3d28/B is initiating minor compaction (all files) 2024-11-16T10:14:46,405 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 8cc2406be4637cf7e35e71b99fcd3d28/A is initiating minor compaction (all files) 2024-11-16T10:14:46,405 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8cc2406be4637cf7e35e71b99fcd3d28/B in TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:46,405 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8cc2406be4637cf7e35e71b99fcd3d28/A in TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:46,405 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/87a2d1b0e1584e398819ce5f67b3d3cb, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/cd23874b66a3441fad6808d08d3fad42, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/4c471a589d68434eb8615ea21ca72829] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp, totalSize=35.2 K 2024-11-16T10:14:46,405 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/517fb434304e492db3a596197c1da6b5, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/b27a0365162c4a8da9d8c3b5af7c3996, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/7ff504bf75cb4b7998ec6a64ca426292] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp, totalSize=99.1 K 2024-11-16T10:14:46,405 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:46,406 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. files: [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/517fb434304e492db3a596197c1da6b5, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/b27a0365162c4a8da9d8c3b5af7c3996, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/7ff504bf75cb4b7998ec6a64ca426292] 2024-11-16T10:14:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,406 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 517fb434304e492db3a596197c1da6b5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731752085386 2024-11-16T10:14:46,406 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 87a2d1b0e1584e398819ce5f67b3d3cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731752085386 2024-11-16T10:14:46,407 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting b27a0365162c4a8da9d8c3b5af7c3996, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731752085434 2024-11-16T10:14:46,407 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting cd23874b66a3441fad6808d08d3fad42, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731752085434 2024-11-16T10:14:46,407 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ff504bf75cb4b7998ec6a64ca426292, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731752086117 2024-11-16T10:14:46,407 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c471a589d68434eb8615ea21ca72829, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731752086117 2024-11-16T10:14:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,427 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,429 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8cc2406be4637cf7e35e71b99fcd3d28#B#compaction#121 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,429 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/ca2bb1ebd94d4d6bacc4ac98bc4da247 is 50, key is test_row_0/B:col10/1731752086117/Put/seqid=0 2024-11-16T10:14:46,429 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116091cc4b583d84565b5a7ec206be099fd_8cc2406be4637cf7e35e71b99fcd3d28 store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,434 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116091cc4b583d84565b5a7ec206be099fd_8cc2406be4637cf7e35e71b99fcd3d28, store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,434 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116091cc4b583d84565b5a7ec206be099fd_8cc2406be4637cf7e35e71b99fcd3d28 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741963_1139 (size=12104) 2024-11-16T10:14:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T10:14:46,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:46,445 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 8cc2406be4637cf7e35e71b99fcd3d28 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T10:14:46,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=A 2024-11-16T10:14:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=B 2024-11-16T10:14:46,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:46,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=C 2024-11-16T10:14:46,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,451 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/ca2bb1ebd94d4d6bacc4ac98bc4da247 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/ca2bb1ebd94d4d6bacc4ac98bc4da247 2024-11-16T10:14:46,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,457 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8cc2406be4637cf7e35e71b99fcd3d28/B of 8cc2406be4637cf7e35e71b99fcd3d28 into ca2bb1ebd94d4d6bacc4ac98bc4da247(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:46,457 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:46,457 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., storeName=8cc2406be4637cf7e35e71b99fcd3d28/B, priority=13, startTime=1731752086404; duration=0sec 2024-11-16T10:14:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,457 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:46,457 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8cc2406be4637cf7e35e71b99fcd3d28:B 2024-11-16T10:14:46,457 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,459 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,459 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 8cc2406be4637cf7e35e71b99fcd3d28/C is initiating minor compaction (all files) 2024-11-16T10:14:46,459 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8cc2406be4637cf7e35e71b99fcd3d28/C in TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:46,459 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/baaf9b3ca37647e4b976997abadb0201, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/3ed2d7f08efa4efc9771fafa04522103, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/7a39fb247ef54b1492e2f6cc096e0669] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp, totalSize=35.2 K 2024-11-16T10:14:46,460 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting baaf9b3ca37647e4b976997abadb0201, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731752085386 2024-11-16T10:14:46,460 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ed2d7f08efa4efc9771fafa04522103, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731752085434 2024-11-16T10:14:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,461 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a39fb247ef54b1492e2f6cc096e0669, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731752086117 2024-11-16T10:14:46,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:46,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:46,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741964_1140 (size=4469) 2024-11-16T10:14:46,474 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8cc2406be4637cf7e35e71b99fcd3d28#A#compaction#120 average throughput is 0.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:46,475 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/cbe848dfc5294fb79967b9e53fc1ad63 is 175, key is test_row_0/A:col10/1731752086117/Put/seqid=0 2024-11-16T10:14:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168809e9c56509457892fa51529cb4f2cf_8cc2406be4637cf7e35e71b99fcd3d28 is 50, key is test_row_1/A:col10/1731752086294/Put/seqid=0 2024-11-16T10:14:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T10:14:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741965_1141 (size=31058) 2024-11-16T10:14:46,515 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8cc2406be4637cf7e35e71b99fcd3d28#C#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:46,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752146507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,516 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/c4015b25a0c64ddaa848a9b02a6cd93d is 50, key is test_row_0/C:col10/1731752086117/Put/seqid=0 2024-11-16T10:14:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752146504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731752086497, max timestamp=1731752086511Num uniques 859; 2024-11-16T10:14:46,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T10:14:46,518 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/cbe848dfc5294fb79967b9e53fc1ad63 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/cbe848dfc5294fb79967b9e53fc1ad63 2024-11-16T10:14:46,525 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8cc2406be4637cf7e35e71b99fcd3d28/A of 8cc2406be4637cf7e35e71b99fcd3d28 into cbe848dfc5294fb79967b9e53fc1ad63(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:46,525 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:46,525 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., storeName=8cc2406be4637cf7e35e71b99fcd3d28/A, priority=13, startTime=1731752086404; duration=0sec 2024-11-16T10:14:46,526 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:46,526 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8cc2406be4637cf7e35e71b99fcd3d28:A 2024-11-16T10:14:46,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752146528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752146531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752146533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741966_1142 (size=9714) 2024-11-16T10:14:46,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741967_1143 (size=12104) 2024-11-16T10:14:46,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752146624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752146624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752146645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752146645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752146645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752146835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752146835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752146855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752146855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:46,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752146855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:46,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:46,963 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168809e9c56509457892fa51529cb4f2cf_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168809e9c56509457892fa51529cb4f2cf_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:46,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/72939a62397c47688209da73a3a4f8e0, store: [table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:46,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/72939a62397c47688209da73a3a4f8e0 is 175, key is test_row_1/A:col10/1731752086294/Put/seqid=0 2024-11-16T10:14:46,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741968_1144 (size=22361) 2024-11-16T10:14:46,979 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/c4015b25a0c64ddaa848a9b02a6cd93d as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/c4015b25a0c64ddaa848a9b02a6cd93d 2024-11-16T10:14:46,984 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8cc2406be4637cf7e35e71b99fcd3d28/C of 8cc2406be4637cf7e35e71b99fcd3d28 into c4015b25a0c64ddaa848a9b02a6cd93d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:46,984 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:46,984 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., storeName=8cc2406be4637cf7e35e71b99fcd3d28/C, priority=13, startTime=1731752086404; duration=0sec 2024-11-16T10:14:46,984 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:46,984 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8cc2406be4637cf7e35e71b99fcd3d28:C 2024-11-16T10:14:47,079 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T10:14:47,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752147144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:47,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:47,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:47,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:47,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752147144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:47,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:47,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:47,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:47,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:47,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:47,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752147164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:47,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:47,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:47,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:47,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:47,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752147164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:47,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:47,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:47,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752147164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:47,371 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=70, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/72939a62397c47688209da73a3a4f8e0 2024-11-16T10:14:47,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/5e0af72fcf124839ab9b049f922d98b2 is 50, key is test_row_1/B:col10/1731752086294/Put/seqid=0 2024-11-16T10:14:47,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741969_1145 (size=9657) 2024-11-16T10:14:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T10:14:47,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752147654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:47,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752147654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:47,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:47,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:47,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:47,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:47,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:47,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:47,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752147674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:47,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:47,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:47,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:47,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752147674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:47,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752147674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:47,786 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/5e0af72fcf124839ab9b049f922d98b2 2024-11-16T10:14:47,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/65ed6eeb565e47ba96375f8da94dc41c is 50, key is test_row_1/C:col10/1731752086294/Put/seqid=0 2024-11-16T10:14:47,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741970_1146 (size=9657) 2024-11-16T10:14:48,216 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/65ed6eeb565e47ba96375f8da94dc41c 2024-11-16T10:14:48,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/72939a62397c47688209da73a3a4f8e0 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/72939a62397c47688209da73a3a4f8e0 2024-11-16T10:14:48,225 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/72939a62397c47688209da73a3a4f8e0, entries=100, sequenceid=70, filesize=21.8 K 2024-11-16T10:14:48,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/5e0af72fcf124839ab9b049f922d98b2 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/5e0af72fcf124839ab9b049f922d98b2 2024-11-16T10:14:48,241 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/5e0af72fcf124839ab9b049f922d98b2, entries=100, sequenceid=70, filesize=9.4 K 2024-11-16T10:14:48,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/65ed6eeb565e47ba96375f8da94dc41c as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/65ed6eeb565e47ba96375f8da94dc41c 2024-11-16T10:14:48,266 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/65ed6eeb565e47ba96375f8da94dc41c, entries=100, sequenceid=70, filesize=9.4 K 2024-11-16T10:14:48,267 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=120.76 KB/123660 for 8cc2406be4637cf7e35e71b99fcd3d28 in 1822ms, sequenceid=70, compaction requested=false 2024-11-16T10:14:48,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:48,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:48,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-16T10:14:48,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-16T10:14:48,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-16T10:14:48,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9130 sec 2024-11-16T10:14:48,274 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.9240 sec 2024-11-16T10:14:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:48,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8cc2406be4637cf7e35e71b99fcd3d28 3/3 column families, dataSize=127.47 KB heapSize=371.88 KB 2024-11-16T10:14:48,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=A 2024-11-16T10:14:48,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:48,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=B 2024-11-16T10:14:48,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:48,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=C 2024-11-16T10:14:48,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:48,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116906ab973b486422aba976503967e00ca_8cc2406be4637cf7e35e71b99fcd3d28 is 50, key is test_row_0/A:col10/1731752086526/Put/seqid=0 2024-11-16T10:14:48,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741971_1147 (size=14594) 2024-11-16T10:14:48,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752148694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:48,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752148703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:48,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752148703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:48,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752148704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:48,708 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_0' in 8cc2406be4637cf7e35e71b99fcd3d28 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:14:48,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:48,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752148705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:48,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752148814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:48,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752148814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:48,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:48,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:48,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752148814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:48,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:48,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:48,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:48,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:48,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:48,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752148814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:48,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:48,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:48,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:49,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:49,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752149035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:49,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:49,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752149035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752149035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:49,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752149035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:49,094 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:49,098 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116906ab973b486422aba976503967e00ca_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116906ab973b486422aba976503967e00ca_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:49,101 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/2b53a90014af45fdb0a3bcbaf7e55185, store: [table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:49,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/2b53a90014af45fdb0a3bcbaf7e55185 is 175, key is test_row_0/A:col10/1731752086526/Put/seqid=0 2024-11-16T10:14:49,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741972_1148 (size=39549) 2024-11-16T10:14:49,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:49,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:49,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752149344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:49,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752149344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:49,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:49,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752149344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:49,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:49,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752149344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:49,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:49,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:49,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:49,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T10:14:49,514 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:14:49,515 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:14:49,556 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/2b53a90014af45fdb0a3bcbaf7e55185 2024-11-16T10:14:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-16T10:14:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T10:14:49,557 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:14:49,558 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:14:49,558 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:14:49,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/5630651d8f584685867bdbea363ee789 is 50, key is test_row_0/B:col10/1731752086526/Put/seqid=0 2024-11-16T10:14:49,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741973_1149 (size=12001) 2024-11-16T10:14:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T10:14:49,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T10:14:49,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:49,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:49,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:49,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:49,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:49,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:49,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T10:14:49,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:49,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:49,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752149864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:49,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:49,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:49,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:49,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752149864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:49,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:49,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:49,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752149864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:49,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:49,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752149864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T10:14:49,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/5630651d8f584685867bdbea363ee789 2024-11-16T10:14:49,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/af28831f55234863a96a46fbaa63ba65 is 50, key is test_row_0/C:col10/1731752086526/Put/seqid=0 2024-11-16T10:14:49,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741974_1150 (size=12001) 2024-11-16T10:14:50,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T10:14:50,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:50,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:50,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:50,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:50,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:50,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T10:14:50,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:50,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:50,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:50,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:50,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:50,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T10:14:50,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T10:14:50,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:50,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:50,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:50,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:50,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:50,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:50,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/af28831f55234863a96a46fbaa63ba65 2024-11-16T10:14:50,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/2b53a90014af45fdb0a3bcbaf7e55185 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/2b53a90014af45fdb0a3bcbaf7e55185 2024-11-16T10:14:50,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/2b53a90014af45fdb0a3bcbaf7e55185, entries=200, sequenceid=99, filesize=38.6 K 2024-11-16T10:14:50,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/5630651d8f584685867bdbea363ee789 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/5630651d8f584685867bdbea363ee789 2024-11-16T10:14:50,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/5630651d8f584685867bdbea363ee789, entries=150, sequenceid=99, filesize=11.7 K 2024-11-16T10:14:50,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/af28831f55234863a96a46fbaa63ba65 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/af28831f55234863a96a46fbaa63ba65 2024-11-16T10:14:50,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/af28831f55234863a96a46fbaa63ba65, entries=150, sequenceid=99, filesize=11.7 K 2024-11-16T10:14:50,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~371.83 KB/380752, currentSize=46.96 KB/48090 for 8cc2406be4637cf7e35e71b99fcd3d28 in 1755ms, sequenceid=99, compaction requested=true 2024-11-16T10:14:50,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:50,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8cc2406be4637cf7e35e71b99fcd3d28:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:14:50,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:50,423 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:50,423 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:50,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8cc2406be4637cf7e35e71b99fcd3d28:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:14:50,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:50,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8cc2406be4637cf7e35e71b99fcd3d28:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:14:50,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:50,424 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:50,425 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 8cc2406be4637cf7e35e71b99fcd3d28/A is initiating minor compaction (all files) 2024-11-16T10:14:50,425 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8cc2406be4637cf7e35e71b99fcd3d28/A in TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:50,425 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/cbe848dfc5294fb79967b9e53fc1ad63, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/72939a62397c47688209da73a3a4f8e0, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/2b53a90014af45fdb0a3bcbaf7e55185] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp, totalSize=90.8 K 2024-11-16T10:14:50,425 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:50,425 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. files: [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/cbe848dfc5294fb79967b9e53fc1ad63, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/72939a62397c47688209da73a3a4f8e0, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/2b53a90014af45fdb0a3bcbaf7e55185] 2024-11-16T10:14:50,425 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:50,425 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 8cc2406be4637cf7e35e71b99fcd3d28/B is initiating minor compaction (all files) 2024-11-16T10:14:50,425 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8cc2406be4637cf7e35e71b99fcd3d28/B in TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:50,425 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/ca2bb1ebd94d4d6bacc4ac98bc4da247, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/5e0af72fcf124839ab9b049f922d98b2, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/5630651d8f584685867bdbea363ee789] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp, totalSize=33.0 K 2024-11-16T10:14:50,426 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting ca2bb1ebd94d4d6bacc4ac98bc4da247, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731752086117 2024-11-16T10:14:50,426 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbe848dfc5294fb79967b9e53fc1ad63, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731752086117 2024-11-16T10:14:50,426 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e0af72fcf124839ab9b049f922d98b2, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731752086294 2024-11-16T10:14:50,426 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72939a62397c47688209da73a3a4f8e0, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731752086294 2024-11-16T10:14:50,427 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 5630651d8f584685867bdbea363ee789, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731752086526 2024-11-16T10:14:50,427 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b53a90014af45fdb0a3bcbaf7e55185, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731752086510 2024-11-16T10:14:50,435 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:50,435 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8cc2406be4637cf7e35e71b99fcd3d28#B#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:50,436 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/ec4f73a1e4714062aeedc26652456acb is 50, key is test_row_0/B:col10/1731752086526/Put/seqid=0 2024-11-16T10:14:50,438 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116e5f3aa72c0c84590be66937e831e2105_8cc2406be4637cf7e35e71b99fcd3d28 store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:50,441 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116e5f3aa72c0c84590be66937e831e2105_8cc2406be4637cf7e35e71b99fcd3d28, store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:50,441 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e5f3aa72c0c84590be66937e831e2105_8cc2406be4637cf7e35e71b99fcd3d28 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:50,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741975_1151 (size=12207) 2024-11-16T10:14:50,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741976_1152 (size=4469) 2024-11-16T10:14:50,452 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8cc2406be4637cf7e35e71b99fcd3d28#A#compaction#130 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:50,452 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/dc26a2a6220046f591478c99a84507dd is 175, key is test_row_0/A:col10/1731752086526/Put/seqid=0 2024-11-16T10:14:50,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741977_1153 (size=31161) 2024-11-16T10:14:50,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T10:14:50,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:50,479 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 8cc2406be4637cf7e35e71b99fcd3d28 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T10:14:50,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=A 2024-11-16T10:14:50,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:50,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=B 2024-11-16T10:14:50,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:50,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=C 2024-11-16T10:14:50,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:50,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116da5fce35adf549fbb6a70f9006a135f3_8cc2406be4637cf7e35e71b99fcd3d28 is 50, key is test_row_0/A:col10/1731752088703/Put/seqid=0 2024-11-16T10:14:50,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741978_1154 (size=12154) 2024-11-16T10:14:50,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:50,512 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116da5fce35adf549fbb6a70f9006a135f3_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116da5fce35adf549fbb6a70f9006a135f3_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:50,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/56a0fa4c805f4c6ea3d14e3909aae2d1, store: [table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:50,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/56a0fa4c805f4c6ea3d14e3909aae2d1 is 175, key is test_row_0/A:col10/1731752088703/Put/seqid=0 2024-11-16T10:14:50,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741979_1155 (size=30955) 2024-11-16T10:14:50,547 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=109, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/56a0fa4c805f4c6ea3d14e3909aae2d1 2024-11-16T10:14:50,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/ceb7478db2b74dd78d8511f5942971e6 is 50, key is test_row_0/B:col10/1731752088703/Put/seqid=0 2024-11-16T10:14:50,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741980_1156 (size=12001) 2024-11-16T10:14:50,578 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/ceb7478db2b74dd78d8511f5942971e6 2024-11-16T10:14:50,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/b4641e4203f143729730f6ef56ba7b8d is 50, key is test_row_0/C:col10/1731752088703/Put/seqid=0 2024-11-16T10:14:50,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741981_1157 (size=12001) 2024-11-16T10:14:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T10:14:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:50,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:50,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752090770, max timestamp=1731752090791Num uniques 859; 2024-11-16T10:14:50,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:14:50,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752090770, max timestamp=1731752090791Num uniques 859; 2024-11-16T10:14:50,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752090770, max timestamp=1731752090791Num uniques 859; 2024-11-16T10:14:50,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:14:50,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:14:50,850 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/ec4f73a1e4714062aeedc26652456acb as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/ec4f73a1e4714062aeedc26652456acb 2024-11-16T10:14:50,857 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8cc2406be4637cf7e35e71b99fcd3d28/B of 8cc2406be4637cf7e35e71b99fcd3d28 into ec4f73a1e4714062aeedc26652456acb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:50,857 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:50,857 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., storeName=8cc2406be4637cf7e35e71b99fcd3d28/B, priority=13, startTime=1731752090423; duration=0sec 2024-11-16T10:14:50,857 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:50,857 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8cc2406be4637cf7e35e71b99fcd3d28:B 2024-11-16T10:14:50,857 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:50,858 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:50,858 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 8cc2406be4637cf7e35e71b99fcd3d28/C is initiating minor compaction (all files) 2024-11-16T10:14:50,858 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8cc2406be4637cf7e35e71b99fcd3d28/C in TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:50,858 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/c4015b25a0c64ddaa848a9b02a6cd93d, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/65ed6eeb565e47ba96375f8da94dc41c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/af28831f55234863a96a46fbaa63ba65] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp, totalSize=33.0 K 2024-11-16T10:14:50,861 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting c4015b25a0c64ddaa848a9b02a6cd93d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731752086117 2024-11-16T10:14:50,861 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 65ed6eeb565e47ba96375f8da94dc41c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731752086294 2024-11-16T10:14:50,862 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting af28831f55234863a96a46fbaa63ba65, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731752086526 2024-11-16T10:14:50,866 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/dc26a2a6220046f591478c99a84507dd as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/dc26a2a6220046f591478c99a84507dd 2024-11-16T10:14:50,872 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8cc2406be4637cf7e35e71b99fcd3d28/A of 8cc2406be4637cf7e35e71b99fcd3d28 into dc26a2a6220046f591478c99a84507dd(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:50,872 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:50,872 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., storeName=8cc2406be4637cf7e35e71b99fcd3d28/A, priority=13, startTime=1731752090423; duration=0sec 2024-11-16T10:14:50,872 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:50,872 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8cc2406be4637cf7e35e71b99fcd3d28:A 2024-11-16T10:14:50,873 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8cc2406be4637cf7e35e71b99fcd3d28#C#compaction#134 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:50,873 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/8f6240085b784530b14894bac5791c66 is 50, key is test_row_0/C:col10/1731752086526/Put/seqid=0 2024-11-16T10:14:50,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:50,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752150866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:50,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:50,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752150874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:50,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:50,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:50,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752150874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:50,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752150874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:50,879 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_2' in 8cc2406be4637cf7e35e71b99fcd3d28 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:14:50,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:50,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:50,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:50,880 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_2' in 8cc2406be4637cf7e35e71b99fcd3d28 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:14:50,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:50,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:50,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:50,880 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_2' in 8cc2406be4637cf7e35e71b99fcd3d28 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:14:50,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:50,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:50,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:50,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741982_1158 (size=12207) 2024-11-16T10:14:50,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752150884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:50,887 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in 8cc2406be4637cf7e35e71b99fcd3d28 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:14:50,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:50,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:50,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:50,890 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/8f6240085b784530b14894bac5791c66 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/8f6240085b784530b14894bac5791c66 2024-11-16T10:14:50,895 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8cc2406be4637cf7e35e71b99fcd3d28/C of 8cc2406be4637cf7e35e71b99fcd3d28 into 8f6240085b784530b14894bac5791c66(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:50,895 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:50,895 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., storeName=8cc2406be4637cf7e35e71b99fcd3d28/C, priority=13, startTime=1731752090423; duration=0sec 2024-11-16T10:14:50,895 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:50,895 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8cc2406be4637cf7e35e71b99fcd3d28:C 2024-11-16T10:14:50,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:50,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752150984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:50,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:50,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:50,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:50,997 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/b4641e4203f143729730f6ef56ba7b8d 2024-11-16T10:14:51,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/56a0fa4c805f4c6ea3d14e3909aae2d1 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/56a0fa4c805f4c6ea3d14e3909aae2d1 2024-11-16T10:14:51,008 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/56a0fa4c805f4c6ea3d14e3909aae2d1, entries=150, sequenceid=109, filesize=30.2 K 2024-11-16T10:14:51,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/ceb7478db2b74dd78d8511f5942971e6 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/ceb7478db2b74dd78d8511f5942971e6 2024-11-16T10:14:51,015 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/ceb7478db2b74dd78d8511f5942971e6, entries=150, sequenceid=109, filesize=11.7 K 2024-11-16T10:14:51,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/b4641e4203f143729730f6ef56ba7b8d as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/b4641e4203f143729730f6ef56ba7b8d 2024-11-16T10:14:51,021 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/b4641e4203f143729730f6ef56ba7b8d, entries=150, sequenceid=109, filesize=11.7 K 2024-11-16T10:14:51,022 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 8cc2406be4637cf7e35e71b99fcd3d28 in 543ms, sequenceid=109, compaction requested=false 2024-11-16T10:14:51,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:51,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:51,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-16T10:14:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-16T10:14:51,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-16T10:14:51,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4660 sec 2024-11-16T10:14:51,027 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.5110 sec 2024-11-16T10:14:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:51,195 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8cc2406be4637cf7e35e71b99fcd3d28 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-16T10:14:51,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=A 2024-11-16T10:14:51,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:51,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=B 2024-11-16T10:14:51,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:51,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=C 2024-11-16T10:14:51,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:51,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b6008d0fe2ad4571adec66b1292d03f0_8cc2406be4637cf7e35e71b99fcd3d28 is 50, key is test_row_0/A:col10/1731752091194/Put/seqid=0 2024-11-16T10:14:51,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741983_1159 (size=14794) 2024-11-16T10:14:51,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752151217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:51,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752151324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:51,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:51,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:51,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:51,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752151534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:51,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:51,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:51,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:51,618 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:51,622 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b6008d0fe2ad4571adec66b1292d03f0_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b6008d0fe2ad4571adec66b1292d03f0_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:51,624 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/87c9a41c7cb04a53bd4b69748df09094, store: [table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:51,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/87c9a41c7cb04a53bd4b69748df09094 is 175, key is test_row_0/A:col10/1731752091194/Put/seqid=0 2024-11-16T10:14:51,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741984_1160 (size=39749) 2024-11-16T10:14:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T10:14:51,704 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:14:51,705 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:14:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-16T10:14:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T10:14:51,706 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:14:51,707 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:14:51,707 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:14:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T10:14:51,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752151845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:51,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:51,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:51,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:51,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T10:14:51,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:51,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:51,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:51,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:51,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:52,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T10:14:52,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:52,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:52,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:52,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:52,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:52,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:52,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T10:14:52,034 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=149, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/87c9a41c7cb04a53bd4b69748df09094 2024-11-16T10:14:52,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/7f8d228c51fb4dd3a96cd13d2993ccbf is 50, key is test_row_0/B:col10/1731752091194/Put/seqid=0 2024-11-16T10:14:52,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741985_1161 (size=12151) 2024-11-16T10:14:52,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T10:14:52,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:52,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:52,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:52,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:52,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:52,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T10:14:52,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:52,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:52,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:52,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:52,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T10:14:52,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752152354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:52,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/7f8d228c51fb4dd3a96cd13d2993ccbf 2024-11-16T10:14:52,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/918b356a24044a7fa8e6a297cbfc389c is 50, key is test_row_0/C:col10/1731752091194/Put/seqid=0 2024-11-16T10:14:52,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741986_1162 (size=12151) 2024-11-16T10:14:52,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/918b356a24044a7fa8e6a297cbfc389c 2024-11-16T10:14:52,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T10:14:52,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:52,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:52,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:52,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:52,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:52,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/87c9a41c7cb04a53bd4b69748df09094 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/87c9a41c7cb04a53bd4b69748df09094 2024-11-16T10:14:52,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/87c9a41c7cb04a53bd4b69748df09094, entries=200, sequenceid=149, filesize=38.8 K 2024-11-16T10:14:52,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/7f8d228c51fb4dd3a96cd13d2993ccbf as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/7f8d228c51fb4dd3a96cd13d2993ccbf 2024-11-16T10:14:52,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/7f8d228c51fb4dd3a96cd13d2993ccbf, entries=150, sequenceid=149, filesize=11.9 K 2024-11-16T10:14:52,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/918b356a24044a7fa8e6a297cbfc389c as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/918b356a24044a7fa8e6a297cbfc389c 2024-11-16T10:14:52,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/918b356a24044a7fa8e6a297cbfc389c, entries=150, sequenceid=149, filesize=11.9 K 2024-11-16T10:14:52,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 8cc2406be4637cf7e35e71b99fcd3d28 in 1341ms, sequenceid=149, compaction requested=true 2024-11-16T10:14:52,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:52,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8cc2406be4637cf7e35e71b99fcd3d28:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:14:52,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:52,537 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:52,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8cc2406be4637cf7e35e71b99fcd3d28:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:14:52,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:52,537 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:52,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8cc2406be4637cf7e35e71b99fcd3d28:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:14:52,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:52,540 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:52,540 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 8cc2406be4637cf7e35e71b99fcd3d28/A is initiating minor compaction (all files) 2024-11-16T10:14:52,540 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:52,540 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 8cc2406be4637cf7e35e71b99fcd3d28/B is initiating minor compaction (all files) 2024-11-16T10:14:52,540 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8cc2406be4637cf7e35e71b99fcd3d28/A in TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:52,540 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8cc2406be4637cf7e35e71b99fcd3d28/B in TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:52,540 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/ec4f73a1e4714062aeedc26652456acb, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/ceb7478db2b74dd78d8511f5942971e6, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/7f8d228c51fb4dd3a96cd13d2993ccbf] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp, totalSize=35.5 K 2024-11-16T10:14:52,540 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/dc26a2a6220046f591478c99a84507dd, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/56a0fa4c805f4c6ea3d14e3909aae2d1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/87c9a41c7cb04a53bd4b69748df09094] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp, totalSize=99.5 K 2024-11-16T10:14:52,540 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:52,540 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. files: [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/dc26a2a6220046f591478c99a84507dd, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/56a0fa4c805f4c6ea3d14e3909aae2d1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/87c9a41c7cb04a53bd4b69748df09094] 2024-11-16T10:14:52,541 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting ec4f73a1e4714062aeedc26652456acb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731752086526 2024-11-16T10:14:52,541 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting ceb7478db2b74dd78d8511f5942971e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731752088675 2024-11-16T10:14:52,544 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc26a2a6220046f591478c99a84507dd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731752086526 2024-11-16T10:14:52,544 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f8d228c51fb4dd3a96cd13d2993ccbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731752090836 2024-11-16T10:14:52,544 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56a0fa4c805f4c6ea3d14e3909aae2d1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731752088675 2024-11-16T10:14:52,545 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87c9a41c7cb04a53bd4b69748df09094, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731752090821 2024-11-16T10:14:52,555 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8cc2406be4637cf7e35e71b99fcd3d28#B#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:52,556 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/f606d91fa4534acbb581cd4038ba312f is 50, key is test_row_0/B:col10/1731752091194/Put/seqid=0 2024-11-16T10:14:52,569 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:52,593 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111642b5482b0b75450b977b985bc0a6adeb_8cc2406be4637cf7e35e71b99fcd3d28 store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:52,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741987_1163 (size=12459) 2024-11-16T10:14:52,595 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111642b5482b0b75450b977b985bc0a6adeb_8cc2406be4637cf7e35e71b99fcd3d28, store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:52,596 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111642b5482b0b75450b977b985bc0a6adeb_8cc2406be4637cf7e35e71b99fcd3d28 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:52,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741988_1164 (size=4469) 2024-11-16T10:14:52,629 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8cc2406be4637cf7e35e71b99fcd3d28#A#compaction#139 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:52,630 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/51877d8c98424beebf300898bef985a0 is 175, key is test_row_0/A:col10/1731752091194/Put/seqid=0 2024-11-16T10:14:52,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T10:14:52,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:52,641 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 8cc2406be4637cf7e35e71b99fcd3d28 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-16T10:14:52,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=A 2024-11-16T10:14:52,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:52,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=B 2024-11-16T10:14:52,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:52,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=C 2024-11-16T10:14:52,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:52,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741989_1165 (size=31413) 2024-11-16T10:14:52,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116773a6e19fb5a4899848b37b70771b8a7_8cc2406be4637cf7e35e71b99fcd3d28 is 50, key is test_row_0/A:col10/1731752091201/Put/seqid=0 2024-11-16T10:14:52,684 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/51877d8c98424beebf300898bef985a0 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/51877d8c98424beebf300898bef985a0 2024-11-16T10:14:52,691 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8cc2406be4637cf7e35e71b99fcd3d28/A of 8cc2406be4637cf7e35e71b99fcd3d28 into 51877d8c98424beebf300898bef985a0(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:52,691 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:52,691 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., storeName=8cc2406be4637cf7e35e71b99fcd3d28/A, priority=13, startTime=1731752092536; duration=0sec 2024-11-16T10:14:52,691 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:52,691 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8cc2406be4637cf7e35e71b99fcd3d28:A 2024-11-16T10:14:52,692 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:52,693 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:52,693 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 8cc2406be4637cf7e35e71b99fcd3d28/C is initiating minor compaction (all files) 2024-11-16T10:14:52,693 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8cc2406be4637cf7e35e71b99fcd3d28/C in TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:52,693 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/8f6240085b784530b14894bac5791c66, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/b4641e4203f143729730f6ef56ba7b8d, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/918b356a24044a7fa8e6a297cbfc389c] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp, totalSize=35.5 K 2024-11-16T10:14:52,693 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f6240085b784530b14894bac5791c66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731752086526 2024-11-16T10:14:52,694 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4641e4203f143729730f6ef56ba7b8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731752088675 2024-11-16T10:14:52,694 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 918b356a24044a7fa8e6a297cbfc389c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731752090836 2024-11-16T10:14:52,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741990_1166 (size=12304) 2024-11-16T10:14:52,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:52,708 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116773a6e19fb5a4899848b37b70771b8a7_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116773a6e19fb5a4899848b37b70771b8a7_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:52,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/22265f11427d42538bc8842953e46c85, store: [table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:52,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/22265f11427d42538bc8842953e46c85 is 175, key is test_row_0/A:col10/1731752091201/Put/seqid=0 2024-11-16T10:14:52,735 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8cc2406be4637cf7e35e71b99fcd3d28#C#compaction#141 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:52,735 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/c82fe738bc9f4d979dd4c9a87f8d94d1 is 50, key is test_row_0/C:col10/1731752091194/Put/seqid=0 2024-11-16T10:14:52,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741991_1167 (size=31105) 2024-11-16T10:14:52,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741992_1168 (size=12459) 2024-11-16T10:14:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T10:14:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:52,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:52,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752092913, max timestamp=1731752092933Num uniques 859; 2024-11-16T10:14:52,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:14:52,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752092913, max timestamp=1731752092933Num uniques 859; 2024-11-16T10:14:52,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:14:52,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752092913, max timestamp=1731752092933Num uniques 859; 2024-11-16T10:14:52,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:14:52,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:52,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752152976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:52,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752152978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:52,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731752092972, max timestamp=1731752092981Num uniques 1018; 2024-11-16T10:14:52,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731752092972, max timestamp=1731752092981Num uniques 1018; 2024-11-16T10:14:52,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-16T10:14:52,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-16T10:14:52,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731752092972, max timestamp=1731752092981Num uniques 1018; 2024-11-16T10:14:52,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-16T10:14:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:53,002 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/f606d91fa4534acbb581cd4038ba312f as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/f606d91fa4534acbb581cd4038ba312f 2024-11-16T10:14:53,009 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8cc2406be4637cf7e35e71b99fcd3d28/B of 8cc2406be4637cf7e35e71b99fcd3d28 into f606d91fa4534acbb581cd4038ba312f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:53,009 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:53,009 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., storeName=8cc2406be4637cf7e35e71b99fcd3d28/B, priority=13, startTime=1731752092537; duration=0sec 2024-11-16T10:14:53,010 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:53,010 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8cc2406be4637cf7e35e71b99fcd3d28:B 2024-11-16T10:14:53,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:53,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:53,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752153061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:53,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752153061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:53,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:53,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:53,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752153094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752153094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:53,144 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/22265f11427d42538bc8842953e46c85 2024-11-16T10:14:53,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/f015a59458524fb59ec6c691107e5ef6 is 50, key is test_row_0/B:col10/1731752091201/Put/seqid=0 2024-11-16T10:14:53,162 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/c82fe738bc9f4d979dd4c9a87f8d94d1 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/c82fe738bc9f4d979dd4c9a87f8d94d1 2024-11-16T10:14:53,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741993_1169 (size=12151) 2024-11-16T10:14:53,166 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/f015a59458524fb59ec6c691107e5ef6 2024-11-16T10:14:53,170 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8cc2406be4637cf7e35e71b99fcd3d28/C of 8cc2406be4637cf7e35e71b99fcd3d28 into c82fe738bc9f4d979dd4c9a87f8d94d1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:53,170 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:53,170 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., storeName=8cc2406be4637cf7e35e71b99fcd3d28/C, priority=13, startTime=1731752092537; duration=0sec 2024-11-16T10:14:53,170 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:53,170 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8cc2406be4637cf7e35e71b99fcd3d28:C 2024-11-16T10:14:53,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/ad6da0fb60b34c49a1a49d12180ac311 is 50, key is test_row_0/C:col10/1731752091201/Put/seqid=0 2024-11-16T10:14:53,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:53,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752153184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:53,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:53,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752153185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:53,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741994_1170 (size=12151) 2024-11-16T10:14:53,195 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/ad6da0fb60b34c49a1a49d12180ac311 2024-11-16T10:14:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/22265f11427d42538bc8842953e46c85 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/22265f11427d42538bc8842953e46c85 2024-11-16T10:14:53,208 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/22265f11427d42538bc8842953e46c85, entries=150, sequenceid=158, filesize=30.4 K 2024-11-16T10:14:53,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/f015a59458524fb59ec6c691107e5ef6 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/f015a59458524fb59ec6c691107e5ef6 2024-11-16T10:14:53,214 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/f015a59458524fb59ec6c691107e5ef6, entries=150, sequenceid=158, filesize=11.9 K 2024-11-16T10:14:53,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/ad6da0fb60b34c49a1a49d12180ac311 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/ad6da0fb60b34c49a1a49d12180ac311 2024-11-16T10:14:53,220 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/ad6da0fb60b34c49a1a49d12180ac311, entries=150, sequenceid=158, filesize=11.9 K 2024-11-16T10:14:53,221 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 8cc2406be4637cf7e35e71b99fcd3d28 in 580ms, sequenceid=158, compaction requested=false 2024-11-16T10:14:53,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:53,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:53,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-16T10:14:53,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-16T10:14:53,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-16T10:14:53,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5170 sec 2024-11-16T10:14:53,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.5270 sec 2024-11-16T10:14:53,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731752093047, max timestamp=1731752093306Num uniques 1077; 2024-11-16T10:14:53,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-16T10:14:53,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731752093047, max timestamp=1731752093306Num uniques 1077; 2024-11-16T10:14:53,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731752093047, max timestamp=1731752093306Num uniques 1077; 2024-11-16T10:14:53,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-16T10:14:53,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-16T10:14:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:53,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8cc2406be4637cf7e35e71b99fcd3d28 3/3 column families, dataSize=53.67 KB heapSize=144.23 KB 2024-11-16T10:14:53,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=A 2024-11-16T10:14:53,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:53,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=B 2024-11-16T10:14:53,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:53,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=C 2024-11-16T10:14:53,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:53,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d7e86f4cef194e9d8ee14aafb38ae2b4_8cc2406be4637cf7e35e71b99fcd3d28 is 50, key is test_row_0/A:col10/1731752093315/Put/seqid=0 2024-11-16T10:14:53,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741995_1171 (size=17284) 2024-11-16T10:14:53,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731752093355, max timestamp=1731752093378Num uniques 859; 2024-11-16T10:14:53,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731752093355, max timestamp=1731752093378Num uniques 859; 2024-11-16T10:14:53,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731752093355, max timestamp=1731752093378Num uniques 859; 2024-11-16T10:14:53,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T10:14:53,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T10:14:53,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T10:14:53,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:53,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752153404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:53,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752153405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:53,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:53,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752153407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:53,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752153408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:53,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:53,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752153410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:53,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:53,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752153525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:53,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752153525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:53,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752153525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:53,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752153525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:53,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:53,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752153525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:53,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:53,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752153734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:53,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752153734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:53,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752153744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:53,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:53,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752153744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:53,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:53,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:53,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752153744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:53,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:53,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:53,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:53,755 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:53,759 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d7e86f4cef194e9d8ee14aafb38ae2b4_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d7e86f4cef194e9d8ee14aafb38ae2b4_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:53,760 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/55c235803c994b16a0f2f1a917734336, store: [table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:53,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/55c235803c994b16a0f2f1a917734336 is 175, key is test_row_0/A:col10/1731752093315/Put/seqid=0 2024-11-16T10:14:53,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741996_1172 (size=48389) 2024-11-16T10:14:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T10:14:53,855 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:14:53,855 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:14:53,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-16T10:14:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T10:14:53,857 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:14:53,857 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:14:53,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:14:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T10:14:54,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T10:14:54,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:54,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:54,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:54,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:54,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:54,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752154044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:54,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:54,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:54,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752154044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:54,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:54,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:54,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:54,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:54,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752154064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:54,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752154064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:54,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752154065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:54,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T10:14:54,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:54,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:54,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:54,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:54,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:54,167 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/55c235803c994b16a0f2f1a917734336 2024-11-16T10:14:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T10:14:54,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/4438d308f23a469a9228526a66efe856 is 50, key is test_row_0/B:col10/1731752093315/Put/seqid=0 2024-11-16T10:14:54,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741997_1173 (size=12151) 2024-11-16T10:14:54,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T10:14:54,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:54,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:54,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:54,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:54,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:54,386 INFO [master/b16ff38458c1:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-16T10:14:54,386 INFO [master/b16ff38458c1:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-16T10:14:54,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T10:14:54,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:54,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:54,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:54,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:54,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T10:14:54,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752154554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:54,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:54,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:54,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:54,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752154554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:54,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:54,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:54,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:54,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/4438d308f23a469a9228526a66efe856 2024-11-16T10:14:54,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752154574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:54,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752154574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:54,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:54,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:54,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:54,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752154574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:54,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T10:14:54,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:54,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:54,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:54,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:54,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:54,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/66280941aa19424bb29fa239234212ee is 50, key is test_row_0/C:col10/1731752093315/Put/seqid=0 2024-11-16T10:14:54,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741998_1174 (size=12151) 2024-11-16T10:14:54,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T10:14:54,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:54,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:54,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:54,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:54,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:54,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T10:14:54,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:54,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:54,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:54,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:54,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T10:14:55,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/66280941aa19424bb29fa239234212ee 2024-11-16T10:14:55,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/55c235803c994b16a0f2f1a917734336 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/55c235803c994b16a0f2f1a917734336 2024-11-16T10:14:55,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/55c235803c994b16a0f2f1a917734336, entries=250, sequenceid=220, filesize=47.3 K 2024-11-16T10:14:55,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/4438d308f23a469a9228526a66efe856 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/4438d308f23a469a9228526a66efe856 2024-11-16T10:14:55,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/4438d308f23a469a9228526a66efe856, entries=150, sequenceid=220, filesize=11.9 K 2024-11-16T10:14:55,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/66280941aa19424bb29fa239234212ee as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/66280941aa19424bb29fa239234212ee 2024-11-16T10:14:55,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/66280941aa19424bb29fa239234212ee, entries=150, sequenceid=220, filesize=11.9 K 2024-11-16T10:14:55,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~144.19 KB/147648, currentSize=134.18 KB/137400 for 8cc2406be4637cf7e35e71b99fcd3d28 in 1733ms, sequenceid=220, compaction requested=true 2024-11-16T10:14:55,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:55,054 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:55,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8cc2406be4637cf7e35e71b99fcd3d28:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:14:55,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:55,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8cc2406be4637cf7e35e71b99fcd3d28:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:14:55,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:55,054 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:55,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8cc2406be4637cf7e35e71b99fcd3d28:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:14:55,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:55,057 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:55,057 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 8cc2406be4637cf7e35e71b99fcd3d28/B is initiating minor compaction (all files) 2024-11-16T10:14:55,057 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8cc2406be4637cf7e35e71b99fcd3d28/B in TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:55,057 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/f606d91fa4534acbb581cd4038ba312f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/f015a59458524fb59ec6c691107e5ef6, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/4438d308f23a469a9228526a66efe856] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp, totalSize=35.9 K 2024-11-16T10:14:55,057 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:55,058 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 8cc2406be4637cf7e35e71b99fcd3d28/A is initiating minor compaction (all files) 2024-11-16T10:14:55,058 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8cc2406be4637cf7e35e71b99fcd3d28/A in TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:55,058 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/51877d8c98424beebf300898bef985a0, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/22265f11427d42538bc8842953e46c85, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/55c235803c994b16a0f2f1a917734336] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp, totalSize=108.3 K 2024-11-16T10:14:55,058 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:55,058 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. files: [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/51877d8c98424beebf300898bef985a0, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/22265f11427d42538bc8842953e46c85, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/55c235803c994b16a0f2f1a917734336] 2024-11-16T10:14:55,058 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting f606d91fa4534acbb581cd4038ba312f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731752090836 2024-11-16T10:14:55,058 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51877d8c98424beebf300898bef985a0, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731752090836 2024-11-16T10:14:55,058 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting f015a59458524fb59ec6c691107e5ef6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731752091201 2024-11-16T10:14:55,059 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22265f11427d42538bc8842953e46c85, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731752091201 2024-11-16T10:14:55,059 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 4438d308f23a469a9228526a66efe856, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731752093057 2024-11-16T10:14:55,059 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55c235803c994b16a0f2f1a917734336, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731752093057 2024-11-16T10:14:55,068 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8cc2406be4637cf7e35e71b99fcd3d28#B#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:55,069 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/c13f9b9592e64ca28b824e7f19961585 is 50, key is test_row_0/B:col10/1731752093315/Put/seqid=0 2024-11-16T10:14:55,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T10:14:55,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:55,082 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 8cc2406be4637cf7e35e71b99fcd3d28 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-16T10:14:55,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=A 2024-11-16T10:14:55,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:55,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=B 2024-11-16T10:14:55,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:55,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=C 2024-11-16T10:14:55,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:14:55,084 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:55,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741999_1175 (size=12561) 2024-11-16T10:14:55,101 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116494eb406d460424fad5e8cbf9889efb9_8cc2406be4637cf7e35e71b99fcd3d28 store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:55,103 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/c13f9b9592e64ca28b824e7f19961585 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/c13f9b9592e64ca28b824e7f19961585 2024-11-16T10:14:55,104 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116494eb406d460424fad5e8cbf9889efb9_8cc2406be4637cf7e35e71b99fcd3d28, store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:55,104 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116494eb406d460424fad5e8cbf9889efb9_8cc2406be4637cf7e35e71b99fcd3d28 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:55,108 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8cc2406be4637cf7e35e71b99fcd3d28/B of 8cc2406be4637cf7e35e71b99fcd3d28 into c13f9b9592e64ca28b824e7f19961585(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:55,108 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:55,108 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., storeName=8cc2406be4637cf7e35e71b99fcd3d28/B, priority=13, startTime=1731752095054; duration=0sec 2024-11-16T10:14:55,108 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:55,108 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8cc2406be4637cf7e35e71b99fcd3d28:B 2024-11-16T10:14:55,108 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:55,110 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:55,110 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 8cc2406be4637cf7e35e71b99fcd3d28/C is initiating minor compaction (all files) 2024-11-16T10:14:55,110 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8cc2406be4637cf7e35e71b99fcd3d28/C in TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:55,110 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/c82fe738bc9f4d979dd4c9a87f8d94d1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/ad6da0fb60b34c49a1a49d12180ac311, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/66280941aa19424bb29fa239234212ee] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp, totalSize=35.9 K 2024-11-16T10:14:55,110 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting c82fe738bc9f4d979dd4c9a87f8d94d1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731752090836 2024-11-16T10:14:55,111 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting ad6da0fb60b34c49a1a49d12180ac311, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731752091201 2024-11-16T10:14:55,111 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 66280941aa19424bb29fa239234212ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731752093057 2024-11-16T10:14:55,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160ecaad8f732a4747ae17712b8f2b361e_8cc2406be4637cf7e35e71b99fcd3d28 is 50, key is test_row_0/A:col10/1731752093404/Put/seqid=0 2024-11-16T10:14:55,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742000_1176 (size=4469) 2024-11-16T10:14:55,126 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8cc2406be4637cf7e35e71b99fcd3d28#C#compaction#150 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:55,126 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/e4584f53e4cd4ac19161ba0e82110317 is 50, key is test_row_0/C:col10/1731752093315/Put/seqid=0 2024-11-16T10:14:55,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742001_1177 (size=12304) 2024-11-16T10:14:55,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742002_1178 (size=12561) 2024-11-16T10:14:55,153 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/e4584f53e4cd4ac19161ba0e82110317 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/e4584f53e4cd4ac19161ba0e82110317 2024-11-16T10:14:55,158 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8cc2406be4637cf7e35e71b99fcd3d28/C of 8cc2406be4637cf7e35e71b99fcd3d28 into e4584f53e4cd4ac19161ba0e82110317(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:55,159 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:55,159 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., storeName=8cc2406be4637cf7e35e71b99fcd3d28/C, priority=13, startTime=1731752095054; duration=0sec 2024-11-16T10:14:55,159 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:55,159 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8cc2406be4637cf7e35e71b99fcd3d28:C 2024-11-16T10:14:55,523 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8cc2406be4637cf7e35e71b99fcd3d28#A#compaction#148 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:55,524 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/f06e4bd9546e4b99b118198012ff9b61 is 175, key is test_row_0/A:col10/1731752093315/Put/seqid=0 2024-11-16T10:14:55,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742003_1179 (size=31515) 2024-11-16T10:14:55,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:55,542 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160ecaad8f732a4747ae17712b8f2b361e_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160ecaad8f732a4747ae17712b8f2b361e_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:55,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/c73f3ba9b4e94c63bb7c6bc4ef4c200e, store: [table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:55,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/c73f3ba9b4e94c63bb7c6bc4ef4c200e is 175, key is test_row_0/A:col10/1731752093404/Put/seqid=0 2024-11-16T10:14:55,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742004_1180 (size=31105) 2024-11-16T10:14:55,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:55,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752155589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:55,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:55,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752155590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752155590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:55,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:55,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752155594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752155594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:55,595 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_0' in 8cc2406be4637cf7e35e71b99fcd3d28 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:14:55,595 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_0' in 8cc2406be4637cf7e35e71b99fcd3d28 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:14:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:55,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752155704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:55,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752155704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:55,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752155704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:55,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752155914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:55,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:55,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752155914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:55,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:55,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:55,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752155914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:55,931 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/f06e4bd9546e4b99b118198012ff9b61 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/f06e4bd9546e4b99b118198012ff9b61 2024-11-16T10:14:55,936 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8cc2406be4637cf7e35e71b99fcd3d28/A of 8cc2406be4637cf7e35e71b99fcd3d28 into f06e4bd9546e4b99b118198012ff9b61(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:55,936 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:55,936 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., storeName=8cc2406be4637cf7e35e71b99fcd3d28/A, priority=13, startTime=1731752095054; duration=0sec 2024-11-16T10:14:55,936 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:55,936 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8cc2406be4637cf7e35e71b99fcd3d28:A 2024-11-16T10:14:55,950 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/c73f3ba9b4e94c63bb7c6bc4ef4c200e 2024-11-16T10:14:55,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/08fb0a6f21d34051a971d5ced226e46d is 50, key is test_row_0/B:col10/1731752093404/Put/seqid=0 2024-11-16T10:14:55,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742005_1181 (size=12151) 2024-11-16T10:14:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T10:14:56,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752156224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:56,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:56,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:56,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752156224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:56,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:56,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:56,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:56,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:56,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752156224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:56,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:56,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:56,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:56,363 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/08fb0a6f21d34051a971d5ced226e46d 2024-11-16T10:14:56,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/fc17b998b8d14261a713387d36254d42 is 50, key is test_row_0/C:col10/1731752093404/Put/seqid=0 2024-11-16T10:14:56,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742006_1182 (size=12151) 2024-11-16T10:14:56,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752156734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:56,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:56,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:56,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:56,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752156734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:56,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:56,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:56,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:56,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752156734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:56,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:56,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:56,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:56,775 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/fc17b998b8d14261a713387d36254d42 2024-11-16T10:14:56,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/c73f3ba9b4e94c63bb7c6bc4ef4c200e as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/c73f3ba9b4e94c63bb7c6bc4ef4c200e 2024-11-16T10:14:56,785 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/c73f3ba9b4e94c63bb7c6bc4ef4c200e, entries=150, sequenceid=254, filesize=30.4 K 2024-11-16T10:14:56,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/08fb0a6f21d34051a971d5ced226e46d as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/08fb0a6f21d34051a971d5ced226e46d 2024-11-16T10:14:56,790 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/08fb0a6f21d34051a971d5ced226e46d, entries=150, sequenceid=254, filesize=11.9 K 2024-11-16T10:14:56,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/fc17b998b8d14261a713387d36254d42 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/fc17b998b8d14261a713387d36254d42 2024-11-16T10:14:56,796 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/fc17b998b8d14261a713387d36254d42, entries=150, sequenceid=254, filesize=11.9 K 2024-11-16T10:14:56,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-16T10:14:56,797 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=60.38 KB/61830 for 8cc2406be4637cf7e35e71b99fcd3d28 in 1715ms, sequenceid=254, compaction requested=false 2024-11-16T10:14:56,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:56,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:56,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-16T10:14:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-16T10:14:56,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-16T10:14:56,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9400 sec 2024-11-16T10:14:56,801 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.9450 sec 2024-11-16T10:14:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:57,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8cc2406be4637cf7e35e71b99fcd3d28 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T10:14:57,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=A 2024-11-16T10:14:57,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:57,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=B 2024-11-16T10:14:57,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:57,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=C 2024-11-16T10:14:57,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:57,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116959667c8113e424c87af30ff152be198_8cc2406be4637cf7e35e71b99fcd3d28 is 50, key is test_row_0/A:col10/1731752097604/Put/seqid=0 2024-11-16T10:14:57,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742007_1183 (size=12454) 2024-11-16T10:14:57,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752157668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:57,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:57,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752157668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:57,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:57,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:57,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:57,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:57,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:57,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:57,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:57,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752157754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:57,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:57,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752157754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:57,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:57,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752157754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:57,759 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_0' in 8cc2406be4637cf7e35e71b99fcd3d28 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:14:57,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:57,759 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in 8cc2406be4637cf7e35e71b99fcd3d28 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:14:57,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:57,760 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in 8cc2406be4637cf7e35e71b99fcd3d28 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:14:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:57,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:57,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752157784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:57,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:57,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:57,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752157784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:57,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:57,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:57,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:57,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:57,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:57,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:57,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752157994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:57,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:57,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752157994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:57,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:57,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:58,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T10:14:58,015 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:14:58,015 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:14:58,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-16T10:14:58,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T10:14:58,016 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:14:58,017 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:14:58,017 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:14:58,022 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:58,032 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116959667c8113e424c87af30ff152be198_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116959667c8113e424c87af30ff152be198_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:58,034 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/3443f17975d74f3a8c2e35c31f86ff30, store: [table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:58,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/3443f17975d74f3a8c2e35c31f86ff30 is 175, key is test_row_0/A:col10/1731752097604/Put/seqid=0 2024-11-16T10:14:58,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742008_1184 (size=31255) 2024-11-16T10:14:58,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T10:14:58,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T10:14:58,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:58,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:58,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:58,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:58,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:58,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:58,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752158304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:58,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:58,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:58,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752158304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:58,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:58,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:58,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:58,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:58,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T10:14:58,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:58,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:58,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:58,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:58,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:58,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T10:14:58,448 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/3443f17975d74f3a8c2e35c31f86ff30 2024-11-16T10:14:58,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/67d6784487ae4cd691021b18aead7075 is 50, key is test_row_0/B:col10/1731752097604/Put/seqid=0 2024-11-16T10:14:58,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742009_1185 (size=12301) 2024-11-16T10:14:58,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T10:14:58,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:58,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:58,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:58,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:58,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:58,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:58,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T10:14:58,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:58,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:58,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:58,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:58,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:58,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T10:14:58,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T10:14:58,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:58,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:58,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:58,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:58,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:58,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:58,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752158824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:58,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:58,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:58,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752158824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:58,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:58,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:58,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:58,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:58,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/67d6784487ae4cd691021b18aead7075 2024-11-16T10:14:58,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/a65c154b62b04b7a806a61a78066f09a is 50, key is test_row_0/C:col10/1731752097604/Put/seqid=0 2024-11-16T10:14:58,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742010_1186 (size=12301) 2024-11-16T10:14:58,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T10:14:58,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:58,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:58,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:58,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:58,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:58,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:59,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T10:14:59,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:59,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:59,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:59,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:59,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:59,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:59,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T10:14:59,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T10:14:59,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:59,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:59,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:59,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:59,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:14:59,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/a65c154b62b04b7a806a61a78066f09a 2024-11-16T10:14:59,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/3443f17975d74f3a8c2e35c31f86ff30 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/3443f17975d74f3a8c2e35c31f86ff30 2024-11-16T10:14:59,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/3443f17975d74f3a8c2e35c31f86ff30, entries=150, sequenceid=270, filesize=30.5 K 2024-11-16T10:14:59,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/67d6784487ae4cd691021b18aead7075 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/67d6784487ae4cd691021b18aead7075 2024-11-16T10:14:59,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/67d6784487ae4cd691021b18aead7075, entries=150, sequenceid=270, filesize=12.0 K 2024-11-16T10:14:59,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/a65c154b62b04b7a806a61a78066f09a as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/a65c154b62b04b7a806a61a78066f09a 2024-11-16T10:14:59,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/a65c154b62b04b7a806a61a78066f09a, entries=150, sequenceid=270, filesize=12.0 K 2024-11-16T10:14:59,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 8cc2406be4637cf7e35e71b99fcd3d28 in 1691ms, sequenceid=270, compaction requested=true 2024-11-16T10:14:59,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:59,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8cc2406be4637cf7e35e71b99fcd3d28:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:14:59,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:59,297 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:59,297 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:59,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8cc2406be4637cf7e35e71b99fcd3d28:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:14:59,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:59,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8cc2406be4637cf7e35e71b99fcd3d28:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:14:59,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:59,298 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:59,298 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93875 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:59,298 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 8cc2406be4637cf7e35e71b99fcd3d28/B is initiating minor compaction (all files) 2024-11-16T10:14:59,298 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 8cc2406be4637cf7e35e71b99fcd3d28/A is initiating minor compaction (all files) 2024-11-16T10:14:59,298 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8cc2406be4637cf7e35e71b99fcd3d28/B in TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:59,298 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8cc2406be4637cf7e35e71b99fcd3d28/A in TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:59,298 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/c13f9b9592e64ca28b824e7f19961585, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/08fb0a6f21d34051a971d5ced226e46d, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/67d6784487ae4cd691021b18aead7075] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp, totalSize=36.1 K 2024-11-16T10:14:59,298 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/f06e4bd9546e4b99b118198012ff9b61, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/c73f3ba9b4e94c63bb7c6bc4ef4c200e, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/3443f17975d74f3a8c2e35c31f86ff30] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp, totalSize=91.7 K 2024-11-16T10:14:59,298 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:59,298 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. files: [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/f06e4bd9546e4b99b118198012ff9b61, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/c73f3ba9b4e94c63bb7c6bc4ef4c200e, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/3443f17975d74f3a8c2e35c31f86ff30] 2024-11-16T10:14:59,299 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting c13f9b9592e64ca28b824e7f19961585, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731752093057 2024-11-16T10:14:59,299 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting f06e4bd9546e4b99b118198012ff9b61, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731752093057 2024-11-16T10:14:59,299 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 08fb0a6f21d34051a971d5ced226e46d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731752093403 2024-11-16T10:14:59,299 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting c73f3ba9b4e94c63bb7c6bc4ef4c200e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731752093403 2024-11-16T10:14:59,299 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 67d6784487ae4cd691021b18aead7075, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731752095581 2024-11-16T10:14:59,299 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3443f17975d74f3a8c2e35c31f86ff30, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731752095581 2024-11-16T10:14:59,305 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:59,307 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8cc2406be4637cf7e35e71b99fcd3d28#B#compaction#157 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:59,308 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/c73a04c5a3b241ee9144b8b13b1dfee9 is 50, key is test_row_0/B:col10/1731752097604/Put/seqid=0 2024-11-16T10:14:59,308 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116c47f20f25ec343678cf257f077d0fd5e_8cc2406be4637cf7e35e71b99fcd3d28 store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:59,310 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116c47f20f25ec343678cf257f077d0fd5e_8cc2406be4637cf7e35e71b99fcd3d28, store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:59,310 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c47f20f25ec343678cf257f077d0fd5e_8cc2406be4637cf7e35e71b99fcd3d28 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:59,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742011_1187 (size=12813) 2024-11-16T10:14:59,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742012_1188 (size=4469) 2024-11-16T10:14:59,335 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8cc2406be4637cf7e35e71b99fcd3d28#A#compaction#156 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:59,335 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/c60c9bd877f5414ea0a48ba27a2b7335 is 175, key is test_row_0/A:col10/1731752097604/Put/seqid=0 2024-11-16T10:14:59,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742013_1189 (size=31767) 2024-11-16T10:14:59,345 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/c60c9bd877f5414ea0a48ba27a2b7335 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/c60c9bd877f5414ea0a48ba27a2b7335 2024-11-16T10:14:59,353 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8cc2406be4637cf7e35e71b99fcd3d28/A of 8cc2406be4637cf7e35e71b99fcd3d28 into c60c9bd877f5414ea0a48ba27a2b7335(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:59,353 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:59,354 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., storeName=8cc2406be4637cf7e35e71b99fcd3d28/A, priority=13, startTime=1731752099297; duration=0sec 2024-11-16T10:14:59,354 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:14:59,354 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8cc2406be4637cf7e35e71b99fcd3d28:A 2024-11-16T10:14:59,354 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:14:59,355 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:14:59,355 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 8cc2406be4637cf7e35e71b99fcd3d28/C is initiating minor compaction (all files) 2024-11-16T10:14:59,355 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8cc2406be4637cf7e35e71b99fcd3d28/C in TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:59,355 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/e4584f53e4cd4ac19161ba0e82110317, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/fc17b998b8d14261a713387d36254d42, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/a65c154b62b04b7a806a61a78066f09a] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp, totalSize=36.1 K 2024-11-16T10:14:59,355 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4584f53e4cd4ac19161ba0e82110317, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731752093057 2024-11-16T10:14:59,356 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc17b998b8d14261a713387d36254d42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731752093403 2024-11-16T10:14:59,359 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting a65c154b62b04b7a806a61a78066f09a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731752095581 2024-11-16T10:14:59,370 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8cc2406be4637cf7e35e71b99fcd3d28#C#compaction#158 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:14:59,370 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/e45093c5d86e4176abfdad97bd69e733 is 50, key is test_row_0/C:col10/1731752097604/Put/seqid=0 2024-11-16T10:14:59,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742014_1190 (size=12813) 2024-11-16T10:14:59,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T10:14:59,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:14:59,393 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 8cc2406be4637cf7e35e71b99fcd3d28 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T10:14:59,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=A 2024-11-16T10:14:59,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:59,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=B 2024-11-16T10:14:59,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:59,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=C 2024-11-16T10:14:59,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:14:59,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ef90da835b6840fa9c1fbb88b4310b0a_8cc2406be4637cf7e35e71b99fcd3d28 is 50, key is test_row_0/A:col10/1731752097654/Put/seqid=0 2024-11-16T10:14:59,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742015_1191 (size=12454) 2024-11-16T10:14:59,729 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/c73a04c5a3b241ee9144b8b13b1dfee9 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/c73a04c5a3b241ee9144b8b13b1dfee9 2024-11-16T10:14:59,734 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8cc2406be4637cf7e35e71b99fcd3d28/B of 8cc2406be4637cf7e35e71b99fcd3d28 into c73a04c5a3b241ee9144b8b13b1dfee9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:59,734 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:59,734 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., storeName=8cc2406be4637cf7e35e71b99fcd3d28/B, priority=13, startTime=1731752099297; duration=0sec 2024-11-16T10:14:59,734 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:59,734 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8cc2406be4637cf7e35e71b99fcd3d28:B 2024-11-16T10:14:59,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:59,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:14:59,780 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/e45093c5d86e4176abfdad97bd69e733 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/e45093c5d86e4176abfdad97bd69e733 2024-11-16T10:14:59,785 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8cc2406be4637cf7e35e71b99fcd3d28/C of 8cc2406be4637cf7e35e71b99fcd3d28 into e45093c5d86e4176abfdad97bd69e733(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:14:59,785 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:14:59,785 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., storeName=8cc2406be4637cf7e35e71b99fcd3d28/C, priority=13, startTime=1731752099298; duration=0sec 2024-11-16T10:14:59,785 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:14:59,785 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8cc2406be4637cf7e35e71b99fcd3d28:C 2024-11-16T10:14:59,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:59,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752159789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:59,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:59,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752159793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:59,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:59,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752159793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:59,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:14:59,818 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ef90da835b6840fa9c1fbb88b4310b0a_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ef90da835b6840fa9c1fbb88b4310b0a_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:14:59,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/7dfcbd6ee30347168edebe90305b8f6a, store: [table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:14:59,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/7dfcbd6ee30347168edebe90305b8f6a is 175, key is test_row_0/A:col10/1731752097654/Put/seqid=0 2024-11-16T10:14:59,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742016_1192 (size=31255) 2024-11-16T10:14:59,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752159844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:59,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752159844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:59,846 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in 8cc2406be4637cf7e35e71b99fcd3d28 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:14:59,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:59,847 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_0' in 8cc2406be4637cf7e35e71b99fcd3d28 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:14:59,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:59,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:59,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:59,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:59,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:59,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752159904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:59,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752159904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:59,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:14:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:14:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752159904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:14:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:14:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:00,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752160114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:00,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:00,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:00,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:00,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752160114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:00,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:00,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:00,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:00,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752160114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:00,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:00,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:00,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T10:15:00,233 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/7dfcbd6ee30347168edebe90305b8f6a 2024-11-16T10:15:00,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/20337e9d8639496b9e3dac76ccbbc438 is 50, key is test_row_0/B:col10/1731752097654/Put/seqid=0 2024-11-16T10:15:00,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742017_1193 (size=12301) 2024-11-16T10:15:00,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752160424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:00,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752160424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:00,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752160425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:00,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:00,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:00,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:00,645 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/20337e9d8639496b9e3dac76ccbbc438 2024-11-16T10:15:00,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/bce7d0024d614873b74581766002d020 is 50, key is test_row_0/C:col10/1731752097654/Put/seqid=0 2024-11-16T10:15:00,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742018_1194 (size=12301) 2024-11-16T10:15:00,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:00,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752160934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:00,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:00,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:00,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:00,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752160934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:00,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:00,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:00,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:00,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:00,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:00,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752160934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:00,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:00,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:00,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:01,061 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/bce7d0024d614873b74581766002d020 2024-11-16T10:15:01,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/7dfcbd6ee30347168edebe90305b8f6a as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/7dfcbd6ee30347168edebe90305b8f6a 2024-11-16T10:15:01,070 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/7dfcbd6ee30347168edebe90305b8f6a, entries=150, sequenceid=291, filesize=30.5 K 2024-11-16T10:15:01,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/20337e9d8639496b9e3dac76ccbbc438 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/20337e9d8639496b9e3dac76ccbbc438 2024-11-16T10:15:01,074 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/20337e9d8639496b9e3dac76ccbbc438, entries=150, sequenceid=291, filesize=12.0 K 2024-11-16T10:15:01,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/bce7d0024d614873b74581766002d020 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/bce7d0024d614873b74581766002d020 2024-11-16T10:15:01,080 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/bce7d0024d614873b74581766002d020, entries=150, sequenceid=291, filesize=12.0 K 2024-11-16T10:15:01,081 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8cc2406be4637cf7e35e71b99fcd3d28 in 1689ms, sequenceid=291, compaction requested=false 2024-11-16T10:15:01,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:15:01,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:01,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-16T10:15:01,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-16T10:15:01,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-16T10:15:01,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0660 sec 2024-11-16T10:15:01,085 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 3.0690 sec 2024-11-16T10:15:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:01,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8cc2406be4637cf7e35e71b99fcd3d28 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T10:15:01,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=A 2024-11-16T10:15:01,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:01,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=B 2024-11-16T10:15:01,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:01,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=C 2024-11-16T10:15:01,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:01,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c6ad3f0a28fd432bb351d0dd8cf75676_8cc2406be4637cf7e35e71b99fcd3d28 is 50, key is test_row_0/A:col10/1731752099791/Put/seqid=0 2024-11-16T10:15:01,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742019_1195 (size=14994) 2024-11-16T10:15:01,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752161903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:01,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:01,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:01,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:01,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752161904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:01,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:01,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:01,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:01,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752161944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:01,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752161944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:01,946 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_2' in 8cc2406be4637cf7e35e71b99fcd3d28 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:15:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:01,946 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_2' in 8cc2406be4637cf7e35e71b99fcd3d28 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:15:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:01,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752161954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:01,958 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in 8cc2406be4637cf7e35e71b99fcd3d28 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:15:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:02,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752162014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:02,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:02,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:02,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:02,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752162014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:02,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:02,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:02,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:02,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T10:15:02,184 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:15:02,185 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:15:02,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-16T10:15:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T10:15:02,186 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:15:02,187 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:15:02,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:15:02,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:02,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752162224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:02,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752162224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:02,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:02,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:02,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:02,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:02,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:02,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:02,282 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:02,285 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c6ad3f0a28fd432bb351d0dd8cf75676_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c6ad3f0a28fd432bb351d0dd8cf75676_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:02,286 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/9cef6a155ecc4f40b48f918dbb353282, store: [table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:15:02,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/9cef6a155ecc4f40b48f918dbb353282 is 175, key is test_row_0/A:col10/1731752099791/Put/seqid=0 2024-11-16T10:15:02,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742020_1196 (size=39949) 2024-11-16T10:15:02,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T10:15:02,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T10:15:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:15:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:02,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:02,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:02,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T10:15:02,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:02,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:15:02,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:02,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:02,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:02,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:02,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T10:15:02,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:02,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752162534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:02,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:02,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:02,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752162534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:02,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:02,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:02,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:02,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:02,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:02,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T10:15:02,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:02,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:15:02,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:02,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:02,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:02,691 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/9cef6a155ecc4f40b48f918dbb353282 2024-11-16T10:15:02,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/67bff51d32304e9dbbf7d50769bd8a60 is 50, key is test_row_0/B:col10/1731752099791/Put/seqid=0 2024-11-16T10:15:02,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742021_1197 (size=12301) 2024-11-16T10:15:02,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T10:15:02,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:02,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:15:02,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:02,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:02,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:02,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T10:15:02,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T10:15:02,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:02,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:15:02,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:02,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:02,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:02,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:03,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752163043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:03,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:03,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752163044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:03,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:03,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:03,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:03,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:03,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:03,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/67bff51d32304e9dbbf7d50769bd8a60 2024-11-16T10:15:03,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T10:15:03,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:03,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:15:03,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:03,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:03,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:03,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/fc548993819f4b8d8032c97d5899c08a is 50, key is test_row_0/C:col10/1731752099791/Put/seqid=0 2024-11-16T10:15:03,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742022_1198 (size=12301) 2024-11-16T10:15:03,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T10:15:03,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:03,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:15:03,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:03,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:03,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T10:15:03,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T10:15:03,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:03,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:15:03,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:03,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:03,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:03,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/fc548993819f4b8d8032c97d5899c08a 2024-11-16T10:15:03,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/9cef6a155ecc4f40b48f918dbb353282 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/9cef6a155ecc4f40b48f918dbb353282 2024-11-16T10:15:03,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/9cef6a155ecc4f40b48f918dbb353282, entries=200, sequenceid=306, filesize=39.0 K 2024-11-16T10:15:03,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/67bff51d32304e9dbbf7d50769bd8a60 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/67bff51d32304e9dbbf7d50769bd8a60 2024-11-16T10:15:03,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/67bff51d32304e9dbbf7d50769bd8a60, entries=150, sequenceid=306, filesize=12.0 K 2024-11-16T10:15:03,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/fc548993819f4b8d8032c97d5899c08a as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/fc548993819f4b8d8032c97d5899c08a 2024-11-16T10:15:03,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/fc548993819f4b8d8032c97d5899c08a, entries=150, sequenceid=306, filesize=12.0 K 2024-11-16T10:15:03,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8cc2406be4637cf7e35e71b99fcd3d28 in 1681ms, sequenceid=306, compaction requested=true 2024-11-16T10:15:03,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:15:03,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8cc2406be4637cf7e35e71b99fcd3d28:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:15:03,546 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:03,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:03,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8cc2406be4637cf7e35e71b99fcd3d28:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:15:03,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:03,546 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:03,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8cc2406be4637cf7e35e71b99fcd3d28:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:15:03,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:03,547 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:03,547 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:03,547 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 8cc2406be4637cf7e35e71b99fcd3d28/A is initiating minor compaction (all files) 2024-11-16T10:15:03,547 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 8cc2406be4637cf7e35e71b99fcd3d28/B is initiating minor compaction (all files) 2024-11-16T10:15:03,547 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8cc2406be4637cf7e35e71b99fcd3d28/A in TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:03,547 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8cc2406be4637cf7e35e71b99fcd3d28/B in TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:03,547 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/c60c9bd877f5414ea0a48ba27a2b7335, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/7dfcbd6ee30347168edebe90305b8f6a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/9cef6a155ecc4f40b48f918dbb353282] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp, totalSize=100.6 K 2024-11-16T10:15:03,547 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/c73a04c5a3b241ee9144b8b13b1dfee9, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/20337e9d8639496b9e3dac76ccbbc438, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/67bff51d32304e9dbbf7d50769bd8a60] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp, totalSize=36.5 K 2024-11-16T10:15:03,547 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:03,547 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. files: [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/c60c9bd877f5414ea0a48ba27a2b7335, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/7dfcbd6ee30347168edebe90305b8f6a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/9cef6a155ecc4f40b48f918dbb353282] 2024-11-16T10:15:03,548 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting c60c9bd877f5414ea0a48ba27a2b7335, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731752095581 2024-11-16T10:15:03,548 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting c73a04c5a3b241ee9144b8b13b1dfee9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731752095581 2024-11-16T10:15:03,548 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dfcbd6ee30347168edebe90305b8f6a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731752097631 2024-11-16T10:15:03,548 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 20337e9d8639496b9e3dac76ccbbc438, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731752097631 2024-11-16T10:15:03,548 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cef6a155ecc4f40b48f918dbb353282, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731752099787 2024-11-16T10:15:03,548 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 67bff51d32304e9dbbf7d50769bd8a60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731752099791 2024-11-16T10:15:03,559 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:15:03,560 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8cc2406be4637cf7e35e71b99fcd3d28#B#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:03,560 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/a7a3ca98257e4cdba58c5c7699dc52d2 is 50, key is test_row_0/B:col10/1731752099791/Put/seqid=0 2024-11-16T10:15:03,565 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411166f145750678d49e98df92e58562dff39_8cc2406be4637cf7e35e71b99fcd3d28 store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:15:03,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T10:15:03,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:03,566 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 8cc2406be4637cf7e35e71b99fcd3d28 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T10:15:03,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=A 2024-11-16T10:15:03,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:03,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=B 2024-11-16T10:15:03,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:03,567 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411166f145750678d49e98df92e58562dff39_8cc2406be4637cf7e35e71b99fcd3d28, store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:15:03,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=C 2024-11-16T10:15:03,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:03,567 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166f145750678d49e98df92e58562dff39_8cc2406be4637cf7e35e71b99fcd3d28 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:15:03,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742023_1199 (size=12915) 2024-11-16T10:15:03,583 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/a7a3ca98257e4cdba58c5c7699dc52d2 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/a7a3ca98257e4cdba58c5c7699dc52d2 2024-11-16T10:15:03,588 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8cc2406be4637cf7e35e71b99fcd3d28/B of 8cc2406be4637cf7e35e71b99fcd3d28 into a7a3ca98257e4cdba58c5c7699dc52d2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:03,588 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:15:03,588 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., storeName=8cc2406be4637cf7e35e71b99fcd3d28/B, priority=13, startTime=1731752103546; duration=0sec 2024-11-16T10:15:03,588 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:03,588 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8cc2406be4637cf7e35e71b99fcd3d28:B 2024-11-16T10:15:03,588 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:03,589 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:03,590 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 8cc2406be4637cf7e35e71b99fcd3d28/C is initiating minor compaction (all files) 2024-11-16T10:15:03,590 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8cc2406be4637cf7e35e71b99fcd3d28/C in TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:03,590 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/e45093c5d86e4176abfdad97bd69e733, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/bce7d0024d614873b74581766002d020, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/fc548993819f4b8d8032c97d5899c08a] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp, totalSize=36.5 K 2024-11-16T10:15:03,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742024_1200 (size=4469) 2024-11-16T10:15:03,590 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting e45093c5d86e4176abfdad97bd69e733, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731752095581 2024-11-16T10:15:03,591 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting bce7d0024d614873b74581766002d020, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731752097631 2024-11-16T10:15:03,591 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting fc548993819f4b8d8032c97d5899c08a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731752099791 2024-11-16T10:15:03,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161b7700e393b24495a67ab9bac6ccc505_8cc2406be4637cf7e35e71b99fcd3d28 is 50, key is test_row_0/A:col10/1731752101902/Put/seqid=0 2024-11-16T10:15:03,594 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8cc2406be4637cf7e35e71b99fcd3d28#A#compaction#166 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:03,595 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/bb19c4122386408facff37fcf30c2058 is 175, key is test_row_0/A:col10/1731752099791/Put/seqid=0 2024-11-16T10:15:03,600 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8cc2406be4637cf7e35e71b99fcd3d28#C#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:03,601 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/daffe8797bb34e87b5634ec297d447f8 is 50, key is test_row_0/C:col10/1731752099791/Put/seqid=0 2024-11-16T10:15:03,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742025_1201 (size=12454) 2024-11-16T10:15:03,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742026_1202 (size=31869) 2024-11-16T10:15:03,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742027_1203 (size=12915) 2024-11-16T10:15:03,624 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/daffe8797bb34e87b5634ec297d447f8 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/daffe8797bb34e87b5634ec297d447f8 2024-11-16T10:15:03,632 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8cc2406be4637cf7e35e71b99fcd3d28/C of 8cc2406be4637cf7e35e71b99fcd3d28 into daffe8797bb34e87b5634ec297d447f8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:03,632 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:15:03,632 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., storeName=8cc2406be4637cf7e35e71b99fcd3d28/C, priority=13, startTime=1731752103546; duration=0sec 2024-11-16T10:15:03,632 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:03,632 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8cc2406be4637cf7e35e71b99fcd3d28:C 2024-11-16T10:15:03,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:03,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. as already flushing 2024-11-16T10:15:03,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:03,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52412 deadline: 1731752163984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:03,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:03,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752163985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:03,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:03,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:03,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:03,986 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in 8cc2406be4637cf7e35e71b99fcd3d28 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:15:03,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:03,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:03,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:03,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752163986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:03,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:03,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:03,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:04,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:04,007 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161b7700e393b24495a67ab9bac6ccc505_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161b7700e393b24495a67ab9bac6ccc505_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:04,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/ef1ddeb790134e5f8b8c2bc18e996000, store: [table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:15:04,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/ef1ddeb790134e5f8b8c2bc18e996000 is 175, key is test_row_0/A:col10/1731752101902/Put/seqid=0 2024-11-16T10:15:04,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742028_1204 (size=31255) 2024-11-16T10:15:04,014 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=325, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/ef1ddeb790134e5f8b8c2bc18e996000 2024-11-16T10:15:04,015 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/bb19c4122386408facff37fcf30c2058 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/bb19c4122386408facff37fcf30c2058 2024-11-16T10:15:04,022 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8cc2406be4637cf7e35e71b99fcd3d28/A of 8cc2406be4637cf7e35e71b99fcd3d28 into bb19c4122386408facff37fcf30c2058(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:04,022 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:15:04,022 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., storeName=8cc2406be4637cf7e35e71b99fcd3d28/A, priority=13, startTime=1731752103546; duration=0sec 2024-11-16T10:15:04,022 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:04,022 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8cc2406be4637cf7e35e71b99fcd3d28:A 2024-11-16T10:15:04,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/c0bb5e8895a0475ca85d1c0e3c2f3902 is 50, key is test_row_0/B:col10/1731752101902/Put/seqid=0 2024-11-16T10:15:04,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742029_1205 (size=12301) 2024-11-16T10:15:04,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:04,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752164054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:04,056 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in 8cc2406be4637cf7e35e71b99fcd3d28 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:15:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:04,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:04,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752164064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:04,066 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_0' in 8cc2406be4637cf7e35e71b99fcd3d28 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:15:04,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:04,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:04,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:04,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:04,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752164094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:04,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:04,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752164094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:04,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:04,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752164317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:04,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:04,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:04,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752164317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:04,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:04,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:04,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:04,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:04,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:04,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T10:15:04,436 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/c0bb5e8895a0475ca85d1c0e3c2f3902 2024-11-16T10:15:04,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/479600a3fe7b4eb1891d0a249e2b89d9 is 50, key is test_row_0/C:col10/1731752101902/Put/seqid=0 2024-11-16T10:15:04,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742030_1206 (size=12301) 2024-11-16T10:15:04,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752164624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:04,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:04,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752164624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:04,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:04,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:04,849 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/479600a3fe7b4eb1891d0a249e2b89d9 2024-11-16T10:15:04,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/ef1ddeb790134e5f8b8c2bc18e996000 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/ef1ddeb790134e5f8b8c2bc18e996000 2024-11-16T10:15:04,904 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/ef1ddeb790134e5f8b8c2bc18e996000, entries=150, sequenceid=325, filesize=30.5 K 2024-11-16T10:15:04,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/c0bb5e8895a0475ca85d1c0e3c2f3902 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/c0bb5e8895a0475ca85d1c0e3c2f3902 2024-11-16T10:15:04,913 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/c0bb5e8895a0475ca85d1c0e3c2f3902, entries=150, sequenceid=325, filesize=12.0 K 2024-11-16T10:15:04,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/479600a3fe7b4eb1891d0a249e2b89d9 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/479600a3fe7b4eb1891d0a249e2b89d9 2024-11-16T10:15:04,918 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/479600a3fe7b4eb1891d0a249e2b89d9, entries=150, sequenceid=325, filesize=12.0 K 2024-11-16T10:15:04,919 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8cc2406be4637cf7e35e71b99fcd3d28 in 1353ms, sequenceid=325, compaction requested=false 2024-11-16T10:15:04,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:15:04,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:04,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-16T10:15:04,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-16T10:15:04,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-16T10:15:04,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7330 sec 2024-11-16T10:15:04,925 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.7380 sec 2024-11-16T10:15:05,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:05,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8cc2406be4637cf7e35e71b99fcd3d28 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T10:15:05,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=A 2024-11-16T10:15:05,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:05,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=B 2024-11-16T10:15:05,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:05,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=C 2024-11-16T10:15:05,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:05,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e8c7f77ad2be4a7ead53791bd70d7347_8cc2406be4637cf7e35e71b99fcd3d28 is 50, key is test_row_0/A:col10/1731752105134/Put/seqid=0 2024-11-16T10:15:05,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742031_1207 (size=12454) 2024-11-16T10:15:05,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752165166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:05,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752165166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:05,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752165274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:05,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752165274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:05,351 INFO [Thread-644 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-644. 2024-11-16T10:15:05,351 DEBUG [Thread-644 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:15:05,351 DEBUG [Thread-644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:05,351 DEBUG [Thread-644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:05,351 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:15:05,353 INFO [Thread-646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-646. 2024-11-16T10:15:05,353 DEBUG [Thread-646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:15:05,353 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:05,353 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:05,353 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:15:05,361 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-11-16T10:15:05,361 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:15:05,361 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:05,361 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:05,361 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:15:05,363 INFO [Thread-648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-648. 2024-11-16T10:15:05,363 DEBUG [Thread-648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:15:05,363 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:05,363 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:05,363 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:15:05,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:05,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752165484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752165484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:05,549 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:05,555 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e8c7f77ad2be4a7ead53791bd70d7347_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e8c7f77ad2be4a7ead53791bd70d7347_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:05,557 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/71bf46d810ac4fb580acdd3da0bb42c5, store: [table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:15:05,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/71bf46d810ac4fb580acdd3da0bb42c5 is 175, key is test_row_0/A:col10/1731752105134/Put/seqid=0 2024-11-16T10:15:05,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742032_1208 (size=31255) 2024-11-16T10:15:05,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:05,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752165795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752165794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:05,964 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/71bf46d810ac4fb580acdd3da0bb42c5 2024-11-16T10:15:05,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/891455a642a74d238a4a2f1da4bc2663 is 50, key is test_row_0/B:col10/1731752105134/Put/seqid=0 2024-11-16T10:15:05,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742033_1209 (size=12301) 2024-11-16T10:15:06,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52512 deadline: 1731752166064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:06,065 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in 8cc2406be4637cf7e35e71b99fcd3d28 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:15:06,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:06,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:06,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:06,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52440 deadline: 1731752166074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:06,075 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_0' in 8cc2406be4637cf7e35e71b99fcd3d28 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:15:06,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:06,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:06,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:06,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:06,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52456 deadline: 1731752166305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52474 deadline: 1731752166305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T10:15:06,346 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:15:06,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/891455a642a74d238a4a2f1da4bc2663 2024-11-16T10:15:06,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/6515f0e060f14cf19347f7c0046a5ef2 is 50, key is test_row_0/C:col10/1731752105134/Put/seqid=0 2024-11-16T10:15:06,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742034_1210 (size=12301) 2024-11-16T10:15:06,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/6515f0e060f14cf19347f7c0046a5ef2 2024-11-16T10:15:06,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/71bf46d810ac4fb580acdd3da0bb42c5 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/71bf46d810ac4fb580acdd3da0bb42c5 2024-11-16T10:15:06,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/71bf46d810ac4fb580acdd3da0bb42c5, entries=150, sequenceid=342, filesize=30.5 K 2024-11-16T10:15:06,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/891455a642a74d238a4a2f1da4bc2663 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/891455a642a74d238a4a2f1da4bc2663 2024-11-16T10:15:06,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/891455a642a74d238a4a2f1da4bc2663, entries=150, sequenceid=342, filesize=12.0 K 2024-11-16T10:15:06,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/6515f0e060f14cf19347f7c0046a5ef2 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/6515f0e060f14cf19347f7c0046a5ef2 2024-11-16T10:15:06,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/6515f0e060f14cf19347f7c0046a5ef2, entries=150, sequenceid=342, filesize=12.0 K 2024-11-16T10:15:06,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8cc2406be4637cf7e35e71b99fcd3d28 in 1690ms, sequenceid=342, compaction requested=true 2024-11-16T10:15:06,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:15:06,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8cc2406be4637cf7e35e71b99fcd3d28:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:15:06,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:06,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8cc2406be4637cf7e35e71b99fcd3d28:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:15:06,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:06,826 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:06,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8cc2406be4637cf7e35e71b99fcd3d28:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:15:06,826 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:06,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:06,828 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:06,828 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:06,828 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 8cc2406be4637cf7e35e71b99fcd3d28/B is initiating minor compaction (all files) 2024-11-16T10:15:06,828 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 8cc2406be4637cf7e35e71b99fcd3d28/A is initiating minor compaction (all files) 2024-11-16T10:15:06,828 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8cc2406be4637cf7e35e71b99fcd3d28/A in TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:06,828 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8cc2406be4637cf7e35e71b99fcd3d28/B in TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:06,828 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/a7a3ca98257e4cdba58c5c7699dc52d2, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/c0bb5e8895a0475ca85d1c0e3c2f3902, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/891455a642a74d238a4a2f1da4bc2663] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp, totalSize=36.6 K 2024-11-16T10:15:06,828 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/bb19c4122386408facff37fcf30c2058, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/ef1ddeb790134e5f8b8c2bc18e996000, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/71bf46d810ac4fb580acdd3da0bb42c5] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp, totalSize=92.2 K 2024-11-16T10:15:06,828 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:06,828 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. files: [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/bb19c4122386408facff37fcf30c2058, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/ef1ddeb790134e5f8b8c2bc18e996000, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/71bf46d810ac4fb580acdd3da0bb42c5] 2024-11-16T10:15:06,828 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting a7a3ca98257e4cdba58c5c7699dc52d2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731752099791 2024-11-16T10:15:06,828 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb19c4122386408facff37fcf30c2058, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731752099791 2024-11-16T10:15:06,829 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting c0bb5e8895a0475ca85d1c0e3c2f3902, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731752101893 2024-11-16T10:15:06,829 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef1ddeb790134e5f8b8c2bc18e996000, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731752101893 2024-11-16T10:15:06,829 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 891455a642a74d238a4a2f1da4bc2663, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731752103979 2024-11-16T10:15:06,829 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71bf46d810ac4fb580acdd3da0bb42c5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731752103979 2024-11-16T10:15:06,839 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8cc2406be4637cf7e35e71b99fcd3d28#B#compaction#174 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:06,839 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:15:06,840 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/6a9a4eecbb134d8fa4200b428c056746 is 50, key is test_row_0/B:col10/1731752105134/Put/seqid=0 2024-11-16T10:15:06,842 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411164fbb7ecc9d734cb5b895a0163163de03_8cc2406be4637cf7e35e71b99fcd3d28 store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:15:06,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742035_1211 (size=13017) 2024-11-16T10:15:06,866 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411164fbb7ecc9d734cb5b895a0163163de03_8cc2406be4637cf7e35e71b99fcd3d28, store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:15:06,867 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164fbb7ecc9d734cb5b895a0163163de03_8cc2406be4637cf7e35e71b99fcd3d28 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:15:06,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742036_1212 (size=4469) 2024-11-16T10:15:07,027 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T10:15:07,250 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/6a9a4eecbb134d8fa4200b428c056746 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/6a9a4eecbb134d8fa4200b428c056746 2024-11-16T10:15:07,256 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8cc2406be4637cf7e35e71b99fcd3d28/B of 8cc2406be4637cf7e35e71b99fcd3d28 into 6a9a4eecbb134d8fa4200b428c056746(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:07,256 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:15:07,256 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., storeName=8cc2406be4637cf7e35e71b99fcd3d28/B, priority=13, startTime=1731752106826; duration=0sec 2024-11-16T10:15:07,256 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:07,256 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8cc2406be4637cf7e35e71b99fcd3d28:B 2024-11-16T10:15:07,257 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:07,258 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:07,258 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 8cc2406be4637cf7e35e71b99fcd3d28/C is initiating minor compaction (all files) 2024-11-16T10:15:07,258 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8cc2406be4637cf7e35e71b99fcd3d28/C in TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:07,258 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/daffe8797bb34e87b5634ec297d447f8, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/479600a3fe7b4eb1891d0a249e2b89d9, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/6515f0e060f14cf19347f7c0046a5ef2] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp, totalSize=36.6 K 2024-11-16T10:15:07,259 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting daffe8797bb34e87b5634ec297d447f8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731752099791 2024-11-16T10:15:07,259 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 479600a3fe7b4eb1891d0a249e2b89d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731752101893 2024-11-16T10:15:07,260 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 6515f0e060f14cf19347f7c0046a5ef2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731752103979 2024-11-16T10:15:07,270 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8cc2406be4637cf7e35e71b99fcd3d28#C#compaction#176 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:07,270 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/44b174ab1ca0487ead9f47cdd633542b is 50, key is test_row_0/C:col10/1731752105134/Put/seqid=0 2024-11-16T10:15:07,272 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8cc2406be4637cf7e35e71b99fcd3d28#A#compaction#175 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:07,272 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/787438cd4678447694559e3d28102789 is 175, key is test_row_0/A:col10/1731752105134/Put/seqid=0 2024-11-16T10:15:07,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742037_1213 (size=13017) 2024-11-16T10:15:07,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742038_1214 (size=31971) 2024-11-16T10:15:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:07,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8cc2406be4637cf7e35e71b99fcd3d28 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T10:15:07,316 INFO [Thread-633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-633. 2024-11-16T10:15:07,316 DEBUG [Thread-633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:15:07,316 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:07,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=A 2024-11-16T10:15:07,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:07,316 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:07,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=B 2024-11-16T10:15:07,316 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:15:07,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:07,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=C 2024-11-16T10:15:07,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:07,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111660499b5633ff465996967ae10b3a38e5_8cc2406be4637cf7e35e71b99fcd3d28 is 50, key is test_row_0/A:col10/1731752107314/Put/seqid=0 2024-11-16T10:15:07,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742039_1215 (size=12454) 2024-11-16T10:15:07,326 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-11-16T10:15:07,326 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:15:07,326 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:07,326 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:07,326 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:15:07,684 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/44b174ab1ca0487ead9f47cdd633542b as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/44b174ab1ca0487ead9f47cdd633542b 2024-11-16T10:15:07,685 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/787438cd4678447694559e3d28102789 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/787438cd4678447694559e3d28102789 2024-11-16T10:15:07,690 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8cc2406be4637cf7e35e71b99fcd3d28/C of 8cc2406be4637cf7e35e71b99fcd3d28 into 44b174ab1ca0487ead9f47cdd633542b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:07,690 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:15:07,690 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., storeName=8cc2406be4637cf7e35e71b99fcd3d28/C, priority=13, startTime=1731752106826; duration=0sec 2024-11-16T10:15:07,690 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:07,690 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8cc2406be4637cf7e35e71b99fcd3d28:C 2024-11-16T10:15:07,691 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8cc2406be4637cf7e35e71b99fcd3d28/A of 8cc2406be4637cf7e35e71b99fcd3d28 into 787438cd4678447694559e3d28102789(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:07,691 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:15:07,691 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., storeName=8cc2406be4637cf7e35e71b99fcd3d28/A, priority=13, startTime=1731752106826; duration=0sec 2024-11-16T10:15:07,691 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:07,691 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8cc2406be4637cf7e35e71b99fcd3d28:A 2024-11-16T10:15:07,726 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:07,730 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111660499b5633ff465996967ae10b3a38e5_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111660499b5633ff465996967ae10b3a38e5_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:07,731 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/9e39b3dfea0b45e68ff47e506fa44feb, store: [table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:15:07,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/9e39b3dfea0b45e68ff47e506fa44feb is 175, key is test_row_0/A:col10/1731752107314/Put/seqid=0 2024-11-16T10:15:07,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742040_1216 (size=31255) 2024-11-16T10:15:07,999 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-11-16T10:15:07,999 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:15:07,999 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:08,000 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:08,000 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:15:08,137 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=362, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/9e39b3dfea0b45e68ff47e506fa44feb 2024-11-16T10:15:08,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/69532d2e934349098c0f4a3531d4dae4 is 50, key is test_row_0/B:col10/1731752107314/Put/seqid=0 2024-11-16T10:15:08,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742041_1217 (size=12301) 2024-11-16T10:15:08,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/69532d2e934349098c0f4a3531d4dae4 2024-11-16T10:15:08,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/e316a9863dd74e00892aff58cc4f6989 is 50, key is test_row_0/C:col10/1731752107314/Put/seqid=0 2024-11-16T10:15:08,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742042_1218 (size=12301) 2024-11-16T10:15:08,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/e316a9863dd74e00892aff58cc4f6989 2024-11-16T10:15:08,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/9e39b3dfea0b45e68ff47e506fa44feb as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/9e39b3dfea0b45e68ff47e506fa44feb 2024-11-16T10:15:08,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/9e39b3dfea0b45e68ff47e506fa44feb, entries=150, sequenceid=362, filesize=30.5 K 2024-11-16T10:15:08,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/69532d2e934349098c0f4a3531d4dae4 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/69532d2e934349098c0f4a3531d4dae4 2024-11-16T10:15:08,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/69532d2e934349098c0f4a3531d4dae4, entries=150, sequenceid=362, filesize=12.0 K 2024-11-16T10:15:08,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/e316a9863dd74e00892aff58cc4f6989 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/e316a9863dd74e00892aff58cc4f6989 2024-11-16T10:15:08,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/e316a9863dd74e00892aff58cc4f6989, entries=150, sequenceid=362, filesize=12.0 K 2024-11-16T10:15:08,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=13.42 KB/13740 for 8cc2406be4637cf7e35e71b99fcd3d28 in 1684ms, sequenceid=362, compaction requested=false 2024-11-16T10:15:08,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8cc2406be4637cf7e35e71b99fcd3d28: 2024-11-16T10:15:10,109 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-11-16T10:15:10,109 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:15:10,109 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:10,110 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:10,110 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:15:10,117 INFO [Thread-641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-641. 2024-11-16T10:15:10,117 DEBUG [Thread-641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:15:10,117 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:10,117 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:10,117 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:15:10,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T10:15:10,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-16T10:15:10,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-16T10:15:10,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-16T10:15:10,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-16T10:15:10,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-16T10:15:10,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T10:15:10,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3160 2024-11-16T10:15:10,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3083 2024-11-16T10:15:10,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T10:15:10,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1296 2024-11-16T10:15:10,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3887 rows 2024-11-16T10:15:10,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1302 2024-11-16T10:15:10,118 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3902 rows 2024-11-16T10:15:10,118 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T10:15:10,118 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T10:15:10,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:10,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:10,118 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:15:10,119 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T10:15:10,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T10:15:10,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T10:15:10,124 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752110124"}]},"ts":"1731752110124"} 2024-11-16T10:15:10,126 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T10:15:10,126 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T10:15:10,127 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T10:15:10,128 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8cc2406be4637cf7e35e71b99fcd3d28, UNASSIGN}] 2024-11-16T10:15:10,129 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8cc2406be4637cf7e35e71b99fcd3d28, UNASSIGN 2024-11-16T10:15:10,130 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=8cc2406be4637cf7e35e71b99fcd3d28, regionState=CLOSING, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:10,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8cc2406be4637cf7e35e71b99fcd3d28, UNASSIGN because future has completed 2024-11-16T10:15:10,132 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T10:15:10,132 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912}] 2024-11-16T10:15:10,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T10:15:10,285 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:10,286 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T10:15:10,286 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing 8cc2406be4637cf7e35e71b99fcd3d28, disabling compactions & flushes 2024-11-16T10:15:10,286 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:10,286 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:10,287 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. after waiting 0 ms 2024-11-16T10:15:10,287 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:10,287 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing 8cc2406be4637cf7e35e71b99fcd3d28 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-16T10:15:10,288 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=A 2024-11-16T10:15:10,288 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:10,288 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=B 2024-11-16T10:15:10,288 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:10,288 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8cc2406be4637cf7e35e71b99fcd3d28, store=C 2024-11-16T10:15:10,288 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:10,298 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111676f9ed8e2d2442c39582c1d963a20736_8cc2406be4637cf7e35e71b99fcd3d28 is 50, key is test_row_0/A:col10/1731752110115/Put/seqid=0 2024-11-16T10:15:10,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742043_1219 (size=9914) 2024-11-16T10:15:10,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T10:15:10,704 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:10,714 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111676f9ed8e2d2442c39582c1d963a20736_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111676f9ed8e2d2442c39582c1d963a20736_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:10,715 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/751b5e0a6f7d4d8e82d6737ab81978b5, store: [table=TestAcidGuarantees family=A region=8cc2406be4637cf7e35e71b99fcd3d28] 2024-11-16T10:15:10,716 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/751b5e0a6f7d4d8e82d6737ab81978b5 is 175, key is test_row_0/A:col10/1731752110115/Put/seqid=0 2024-11-16T10:15:10,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742044_1220 (size=22561) 2024-11-16T10:15:10,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T10:15:11,123 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=371, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/751b5e0a6f7d4d8e82d6737ab81978b5 2024-11-16T10:15:11,137 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/a3e0347451f0459bb60657ba2273ed12 is 50, key is test_row_0/B:col10/1731752110115/Put/seqid=0 2024-11-16T10:15:11,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742045_1221 (size=9857) 2024-11-16T10:15:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T10:15:11,541 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/a3e0347451f0459bb60657ba2273ed12 2024-11-16T10:15:11,550 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/fbc460dcf86a407ea6712840122f9f8b is 50, key is test_row_0/C:col10/1731752110115/Put/seqid=0 2024-11-16T10:15:11,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742046_1222 (size=9857) 2024-11-16T10:15:11,955 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/fbc460dcf86a407ea6712840122f9f8b 2024-11-16T10:15:11,959 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/A/751b5e0a6f7d4d8e82d6737ab81978b5 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/751b5e0a6f7d4d8e82d6737ab81978b5 2024-11-16T10:15:11,963 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/751b5e0a6f7d4d8e82d6737ab81978b5, entries=100, sequenceid=371, filesize=22.0 K 2024-11-16T10:15:11,964 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/B/a3e0347451f0459bb60657ba2273ed12 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/a3e0347451f0459bb60657ba2273ed12 2024-11-16T10:15:11,968 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/a3e0347451f0459bb60657ba2273ed12, entries=100, sequenceid=371, filesize=9.6 K 2024-11-16T10:15:11,969 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/.tmp/C/fbc460dcf86a407ea6712840122f9f8b as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/fbc460dcf86a407ea6712840122f9f8b 2024-11-16T10:15:11,973 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/fbc460dcf86a407ea6712840122f9f8b, entries=100, sequenceid=371, filesize=9.6 K 2024-11-16T10:15:11,973 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 8cc2406be4637cf7e35e71b99fcd3d28 in 1686ms, sequenceid=371, compaction requested=true 2024-11-16T10:15:11,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/517fb434304e492db3a596197c1da6b5, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/b27a0365162c4a8da9d8c3b5af7c3996, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/7ff504bf75cb4b7998ec6a64ca426292, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/cbe848dfc5294fb79967b9e53fc1ad63, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/72939a62397c47688209da73a3a4f8e0, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/2b53a90014af45fdb0a3bcbaf7e55185, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/dc26a2a6220046f591478c99a84507dd, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/56a0fa4c805f4c6ea3d14e3909aae2d1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/87c9a41c7cb04a53bd4b69748df09094, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/51877d8c98424beebf300898bef985a0, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/22265f11427d42538bc8842953e46c85, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/55c235803c994b16a0f2f1a917734336, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/f06e4bd9546e4b99b118198012ff9b61, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/c73f3ba9b4e94c63bb7c6bc4ef4c200e, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/c60c9bd877f5414ea0a48ba27a2b7335, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/3443f17975d74f3a8c2e35c31f86ff30, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/7dfcbd6ee30347168edebe90305b8f6a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/9cef6a155ecc4f40b48f918dbb353282, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/bb19c4122386408facff37fcf30c2058, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/ef1ddeb790134e5f8b8c2bc18e996000, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/71bf46d810ac4fb580acdd3da0bb42c5] to archive 2024-11-16T10:15:11,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:15:11,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/517fb434304e492db3a596197c1da6b5 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/517fb434304e492db3a596197c1da6b5 2024-11-16T10:15:11,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/b27a0365162c4a8da9d8c3b5af7c3996 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/b27a0365162c4a8da9d8c3b5af7c3996 2024-11-16T10:15:11,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/7ff504bf75cb4b7998ec6a64ca426292 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/7ff504bf75cb4b7998ec6a64ca426292 2024-11-16T10:15:11,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/cbe848dfc5294fb79967b9e53fc1ad63 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/cbe848dfc5294fb79967b9e53fc1ad63 2024-11-16T10:15:11,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/72939a62397c47688209da73a3a4f8e0 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/72939a62397c47688209da73a3a4f8e0 2024-11-16T10:15:11,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/2b53a90014af45fdb0a3bcbaf7e55185 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/2b53a90014af45fdb0a3bcbaf7e55185 2024-11-16T10:15:11,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/dc26a2a6220046f591478c99a84507dd to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/dc26a2a6220046f591478c99a84507dd 2024-11-16T10:15:11,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/56a0fa4c805f4c6ea3d14e3909aae2d1 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/56a0fa4c805f4c6ea3d14e3909aae2d1 2024-11-16T10:15:11,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/87c9a41c7cb04a53bd4b69748df09094 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/87c9a41c7cb04a53bd4b69748df09094 2024-11-16T10:15:11,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/51877d8c98424beebf300898bef985a0 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/51877d8c98424beebf300898bef985a0 2024-11-16T10:15:11,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/22265f11427d42538bc8842953e46c85 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/22265f11427d42538bc8842953e46c85 2024-11-16T10:15:11,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/55c235803c994b16a0f2f1a917734336 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/55c235803c994b16a0f2f1a917734336 2024-11-16T10:15:11,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/f06e4bd9546e4b99b118198012ff9b61 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/f06e4bd9546e4b99b118198012ff9b61 2024-11-16T10:15:11,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/c73f3ba9b4e94c63bb7c6bc4ef4c200e to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/c73f3ba9b4e94c63bb7c6bc4ef4c200e 2024-11-16T10:15:11,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/c60c9bd877f5414ea0a48ba27a2b7335 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/c60c9bd877f5414ea0a48ba27a2b7335 2024-11-16T10:15:11,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/3443f17975d74f3a8c2e35c31f86ff30 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/3443f17975d74f3a8c2e35c31f86ff30 2024-11-16T10:15:11,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/7dfcbd6ee30347168edebe90305b8f6a to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/7dfcbd6ee30347168edebe90305b8f6a 2024-11-16T10:15:11,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/9cef6a155ecc4f40b48f918dbb353282 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/9cef6a155ecc4f40b48f918dbb353282 2024-11-16T10:15:11,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/bb19c4122386408facff37fcf30c2058 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/bb19c4122386408facff37fcf30c2058 2024-11-16T10:15:11,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/ef1ddeb790134e5f8b8c2bc18e996000 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/ef1ddeb790134e5f8b8c2bc18e996000 2024-11-16T10:15:11,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/71bf46d810ac4fb580acdd3da0bb42c5 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/71bf46d810ac4fb580acdd3da0bb42c5 2024-11-16T10:15:11,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/87a2d1b0e1584e398819ce5f67b3d3cb, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/cd23874b66a3441fad6808d08d3fad42, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/ca2bb1ebd94d4d6bacc4ac98bc4da247, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/4c471a589d68434eb8615ea21ca72829, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/5e0af72fcf124839ab9b049f922d98b2, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/ec4f73a1e4714062aeedc26652456acb, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/5630651d8f584685867bdbea363ee789, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/ceb7478db2b74dd78d8511f5942971e6, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/f606d91fa4534acbb581cd4038ba312f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/7f8d228c51fb4dd3a96cd13d2993ccbf, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/f015a59458524fb59ec6c691107e5ef6, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/c13f9b9592e64ca28b824e7f19961585, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/4438d308f23a469a9228526a66efe856, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/08fb0a6f21d34051a971d5ced226e46d, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/c73a04c5a3b241ee9144b8b13b1dfee9, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/67d6784487ae4cd691021b18aead7075, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/20337e9d8639496b9e3dac76ccbbc438, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/a7a3ca98257e4cdba58c5c7699dc52d2, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/67bff51d32304e9dbbf7d50769bd8a60, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/c0bb5e8895a0475ca85d1c0e3c2f3902, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/891455a642a74d238a4a2f1da4bc2663] to archive 2024-11-16T10:15:11,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:15:11,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/87a2d1b0e1584e398819ce5f67b3d3cb to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/87a2d1b0e1584e398819ce5f67b3d3cb 2024-11-16T10:15:11,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/cd23874b66a3441fad6808d08d3fad42 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/cd23874b66a3441fad6808d08d3fad42 2024-11-16T10:15:11,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/ca2bb1ebd94d4d6bacc4ac98bc4da247 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/ca2bb1ebd94d4d6bacc4ac98bc4da247 2024-11-16T10:15:11,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/4c471a589d68434eb8615ea21ca72829 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/4c471a589d68434eb8615ea21ca72829 2024-11-16T10:15:12,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/5e0af72fcf124839ab9b049f922d98b2 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/5e0af72fcf124839ab9b049f922d98b2 2024-11-16T10:15:12,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/ec4f73a1e4714062aeedc26652456acb to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/ec4f73a1e4714062aeedc26652456acb 2024-11-16T10:15:12,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/5630651d8f584685867bdbea363ee789 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/5630651d8f584685867bdbea363ee789 2024-11-16T10:15:12,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/ceb7478db2b74dd78d8511f5942971e6 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/ceb7478db2b74dd78d8511f5942971e6 2024-11-16T10:15:12,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/f606d91fa4534acbb581cd4038ba312f to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/f606d91fa4534acbb581cd4038ba312f 2024-11-16T10:15:12,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/7f8d228c51fb4dd3a96cd13d2993ccbf to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/7f8d228c51fb4dd3a96cd13d2993ccbf 2024-11-16T10:15:12,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/f015a59458524fb59ec6c691107e5ef6 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/f015a59458524fb59ec6c691107e5ef6 2024-11-16T10:15:12,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/c13f9b9592e64ca28b824e7f19961585 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/c13f9b9592e64ca28b824e7f19961585 2024-11-16T10:15:12,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/4438d308f23a469a9228526a66efe856 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/4438d308f23a469a9228526a66efe856 2024-11-16T10:15:12,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/08fb0a6f21d34051a971d5ced226e46d to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/08fb0a6f21d34051a971d5ced226e46d 2024-11-16T10:15:12,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/c73a04c5a3b241ee9144b8b13b1dfee9 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/c73a04c5a3b241ee9144b8b13b1dfee9 2024-11-16T10:15:12,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/67d6784487ae4cd691021b18aead7075 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/67d6784487ae4cd691021b18aead7075 2024-11-16T10:15:12,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/20337e9d8639496b9e3dac76ccbbc438 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/20337e9d8639496b9e3dac76ccbbc438 2024-11-16T10:15:12,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/a7a3ca98257e4cdba58c5c7699dc52d2 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/a7a3ca98257e4cdba58c5c7699dc52d2 2024-11-16T10:15:12,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/67bff51d32304e9dbbf7d50769bd8a60 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/67bff51d32304e9dbbf7d50769bd8a60 2024-11-16T10:15:12,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/c0bb5e8895a0475ca85d1c0e3c2f3902 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/c0bb5e8895a0475ca85d1c0e3c2f3902 2024-11-16T10:15:12,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/891455a642a74d238a4a2f1da4bc2663 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/891455a642a74d238a4a2f1da4bc2663 2024-11-16T10:15:12,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/baaf9b3ca37647e4b976997abadb0201, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/3ed2d7f08efa4efc9771fafa04522103, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/c4015b25a0c64ddaa848a9b02a6cd93d, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/7a39fb247ef54b1492e2f6cc096e0669, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/65ed6eeb565e47ba96375f8da94dc41c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/8f6240085b784530b14894bac5791c66, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/af28831f55234863a96a46fbaa63ba65, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/b4641e4203f143729730f6ef56ba7b8d, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/c82fe738bc9f4d979dd4c9a87f8d94d1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/918b356a24044a7fa8e6a297cbfc389c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/ad6da0fb60b34c49a1a49d12180ac311, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/e4584f53e4cd4ac19161ba0e82110317, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/66280941aa19424bb29fa239234212ee, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/fc17b998b8d14261a713387d36254d42, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/e45093c5d86e4176abfdad97bd69e733, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/a65c154b62b04b7a806a61a78066f09a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/bce7d0024d614873b74581766002d020, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/daffe8797bb34e87b5634ec297d447f8, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/fc548993819f4b8d8032c97d5899c08a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/479600a3fe7b4eb1891d0a249e2b89d9, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/6515f0e060f14cf19347f7c0046a5ef2] to archive 2024-11-16T10:15:12,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:15:12,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/baaf9b3ca37647e4b976997abadb0201 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/baaf9b3ca37647e4b976997abadb0201 2024-11-16T10:15:12,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/3ed2d7f08efa4efc9771fafa04522103 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/3ed2d7f08efa4efc9771fafa04522103 2024-11-16T10:15:12,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/c4015b25a0c64ddaa848a9b02a6cd93d to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/c4015b25a0c64ddaa848a9b02a6cd93d 2024-11-16T10:15:12,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/7a39fb247ef54b1492e2f6cc096e0669 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/7a39fb247ef54b1492e2f6cc096e0669 2024-11-16T10:15:12,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/65ed6eeb565e47ba96375f8da94dc41c to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/65ed6eeb565e47ba96375f8da94dc41c 2024-11-16T10:15:12,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/8f6240085b784530b14894bac5791c66 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/8f6240085b784530b14894bac5791c66 2024-11-16T10:15:12,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/af28831f55234863a96a46fbaa63ba65 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/af28831f55234863a96a46fbaa63ba65 2024-11-16T10:15:12,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/b4641e4203f143729730f6ef56ba7b8d to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/b4641e4203f143729730f6ef56ba7b8d 2024-11-16T10:15:12,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/c82fe738bc9f4d979dd4c9a87f8d94d1 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/c82fe738bc9f4d979dd4c9a87f8d94d1 2024-11-16T10:15:12,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/918b356a24044a7fa8e6a297cbfc389c to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/918b356a24044a7fa8e6a297cbfc389c 2024-11-16T10:15:12,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/ad6da0fb60b34c49a1a49d12180ac311 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/ad6da0fb60b34c49a1a49d12180ac311 2024-11-16T10:15:12,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/e4584f53e4cd4ac19161ba0e82110317 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/e4584f53e4cd4ac19161ba0e82110317 2024-11-16T10:15:12,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/66280941aa19424bb29fa239234212ee to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/66280941aa19424bb29fa239234212ee 2024-11-16T10:15:12,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/fc17b998b8d14261a713387d36254d42 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/fc17b998b8d14261a713387d36254d42 2024-11-16T10:15:12,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/e45093c5d86e4176abfdad97bd69e733 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/e45093c5d86e4176abfdad97bd69e733 2024-11-16T10:15:12,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/a65c154b62b04b7a806a61a78066f09a to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/a65c154b62b04b7a806a61a78066f09a 2024-11-16T10:15:12,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/bce7d0024d614873b74581766002d020 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/bce7d0024d614873b74581766002d020 2024-11-16T10:15:12,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/daffe8797bb34e87b5634ec297d447f8 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/daffe8797bb34e87b5634ec297d447f8 2024-11-16T10:15:12,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/fc548993819f4b8d8032c97d5899c08a to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/fc548993819f4b8d8032c97d5899c08a 2024-11-16T10:15:12,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/479600a3fe7b4eb1891d0a249e2b89d9 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/479600a3fe7b4eb1891d0a249e2b89d9 2024-11-16T10:15:12,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/6515f0e060f14cf19347f7c0046a5ef2 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/6515f0e060f14cf19347f7c0046a5ef2 2024-11-16T10:15:12,037 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/recovered.edits/374.seqid, newMaxSeqId=374, maxSeqId=4 2024-11-16T10:15:12,038 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. 2024-11-16T10:15:12,038 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for 8cc2406be4637cf7e35e71b99fcd3d28: Waiting for close lock at 1731752110286Running coprocessor pre-close hooks at 1731752110286Disabling compacts and flushes for region at 1731752110286Disabling writes for close at 1731752110287 (+1 ms)Obtaining lock to block concurrent updates at 1731752110287Preparing flush snapshotting stores in 8cc2406be4637cf7e35e71b99fcd3d28 at 1731752110287Finished memstore snapshotting TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731752110289 (+2 ms)Flushing stores of TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28. at 1731752110290 (+1 ms)Flushing 8cc2406be4637cf7e35e71b99fcd3d28/A: creating writer at 1731752110290Flushing mob file 8cc2406be4637cf7e35e71b99fcd3d28/A: appending metadata at 1731752110298 (+8 ms)Flushing mob file 8cc2406be4637cf7e35e71b99fcd3d28/A: closing flushed file at 1731752110298Flushing 8cc2406be4637cf7e35e71b99fcd3d28/A: appending metadata at 1731752110715 (+417 ms)Flushing 8cc2406be4637cf7e35e71b99fcd3d28/A: closing flushed file at 1731752110715Flushing 8cc2406be4637cf7e35e71b99fcd3d28/B: creating writer at 1731752111133 (+418 ms)Flushing 8cc2406be4637cf7e35e71b99fcd3d28/B: appending metadata at 1731752111136 (+3 ms)Flushing 8cc2406be4637cf7e35e71b99fcd3d28/B: closing flushed file at 1731752111136Flushing 8cc2406be4637cf7e35e71b99fcd3d28/C: creating writer at 1731752111546 (+410 ms)Flushing 8cc2406be4637cf7e35e71b99fcd3d28/C: appending metadata at 1731752111549 (+3 ms)Flushing 8cc2406be4637cf7e35e71b99fcd3d28/C: closing flushed file at 1731752111549Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25987fbe: reopening flushed file at 1731752111959 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ae48507: reopening flushed file at 1731752111963 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48614ddb: reopening flushed file at 1731752111968 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 8cc2406be4637cf7e35e71b99fcd3d28 in 1686ms, sequenceid=371, compaction requested=true at 1731752111973 (+5 ms)Writing region close event to WAL at 1731752112034 (+61 ms)Running coprocessor post-close hooks at 1731752112038 (+4 ms)Closed at 1731752112038 2024-11-16T10:15:12,040 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed 8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:12,040 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=8cc2406be4637cf7e35e71b99fcd3d28, regionState=CLOSED 2024-11-16T10:15:12,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 because future has completed 2024-11-16T10:15:12,043 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-16T10:15:12,043 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure 8cc2406be4637cf7e35e71b99fcd3d28, server=b16ff38458c1,45417,1731752048912 in 1.9100 sec 2024-11-16T10:15:12,045 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-16T10:15:12,045 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8cc2406be4637cf7e35e71b99fcd3d28, UNASSIGN in 1.9150 sec 2024-11-16T10:15:12,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-16T10:15:12,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9180 sec 2024-11-16T10:15:12,048 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752112048"}]},"ts":"1731752112048"} 2024-11-16T10:15:12,049 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T10:15:12,049 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T10:15:12,051 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9310 sec 2024-11-16T10:15:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T10:15:12,275 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:15:12,275 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T10:15:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:15:12,277 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:15:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-16T10:15:12,279 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:15:12,282 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:12,287 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A, FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B, FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C, FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/recovered.edits] 2024-11-16T10:15:12,290 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/751b5e0a6f7d4d8e82d6737ab81978b5 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/751b5e0a6f7d4d8e82d6737ab81978b5 2024-11-16T10:15:12,292 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/787438cd4678447694559e3d28102789 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/787438cd4678447694559e3d28102789 2024-11-16T10:15:12,293 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/9e39b3dfea0b45e68ff47e506fa44feb to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/A/9e39b3dfea0b45e68ff47e506fa44feb 2024-11-16T10:15:12,295 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/69532d2e934349098c0f4a3531d4dae4 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/69532d2e934349098c0f4a3531d4dae4 2024-11-16T10:15:12,296 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/6a9a4eecbb134d8fa4200b428c056746 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/6a9a4eecbb134d8fa4200b428c056746 2024-11-16T10:15:12,298 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/a3e0347451f0459bb60657ba2273ed12 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/B/a3e0347451f0459bb60657ba2273ed12 2024-11-16T10:15:12,300 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/44b174ab1ca0487ead9f47cdd633542b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/44b174ab1ca0487ead9f47cdd633542b 2024-11-16T10:15:12,301 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/e316a9863dd74e00892aff58cc4f6989 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/e316a9863dd74e00892aff58cc4f6989 2024-11-16T10:15:12,302 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/fbc460dcf86a407ea6712840122f9f8b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/C/fbc460dcf86a407ea6712840122f9f8b 2024-11-16T10:15:12,304 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/recovered.edits/374.seqid to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28/recovered.edits/374.seqid 2024-11-16T10:15:12,305 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:12,305 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T10:15:12,305 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T10:15:12,306 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T10:15:12,309 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160ecaad8f732a4747ae17712b8f2b361e_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160ecaad8f732a4747ae17712b8f2b361e_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:12,310 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161b7700e393b24495a67ab9bac6ccc505_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161b7700e393b24495a67ab9bac6ccc505_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:12,311 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116493f5b3e553d4900836f3dfce857fedc_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116493f5b3e553d4900836f3dfce857fedc_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:12,312 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111660499b5633ff465996967ae10b3a38e5_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111660499b5633ff465996967ae10b3a38e5_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:12,313 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111676f9ed8e2d2442c39582c1d963a20736_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111676f9ed8e2d2442c39582c1d963a20736_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:12,314 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116773a6e19fb5a4899848b37b70771b8a7_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116773a6e19fb5a4899848b37b70771b8a7_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:12,315 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168784dbcd7b344652886c55c7c1fee759_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168784dbcd7b344652886c55c7c1fee759_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:12,316 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168809e9c56509457892fa51529cb4f2cf_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168809e9c56509457892fa51529cb4f2cf_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:12,317 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116906ab973b486422aba976503967e00ca_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116906ab973b486422aba976503967e00ca_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:12,318 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116959667c8113e424c87af30ff152be198_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116959667c8113e424c87af30ff152be198_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:12,319 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169912af47b8ed418ca9dc9d7b2f4dc7c3_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169912af47b8ed418ca9dc9d7b2f4dc7c3_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:12,320 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b6008d0fe2ad4571adec66b1292d03f0_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b6008d0fe2ad4571adec66b1292d03f0_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:12,322 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c6ad3f0a28fd432bb351d0dd8cf75676_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c6ad3f0a28fd432bb351d0dd8cf75676_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:12,323 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d7e86f4cef194e9d8ee14aafb38ae2b4_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d7e86f4cef194e9d8ee14aafb38ae2b4_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:12,324 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116da5fce35adf549fbb6a70f9006a135f3_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116da5fce35adf549fbb6a70f9006a135f3_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:12,325 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e8c7f77ad2be4a7ead53791bd70d7347_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e8c7f77ad2be4a7ead53791bd70d7347_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:12,326 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ef90da835b6840fa9c1fbb88b4310b0a_8cc2406be4637cf7e35e71b99fcd3d28 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ef90da835b6840fa9c1fbb88b4310b0a_8cc2406be4637cf7e35e71b99fcd3d28 2024-11-16T10:15:12,327 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T10:15:12,329 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:15:12,331 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T10:15:12,333 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T10:15:12,335 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:15:12,335 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T10:15:12,335 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731752112335"}]},"ts":"9223372036854775807"} 2024-11-16T10:15:12,337 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T10:15:12,337 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8cc2406be4637cf7e35e71b99fcd3d28, NAME => 'TestAcidGuarantees,,1731752082170.8cc2406be4637cf7e35e71b99fcd3d28.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T10:15:12,337 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T10:15:12,337 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731752112337"}]},"ts":"9223372036854775807"} 2024-11-16T10:15:12,339 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T10:15:12,340 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:15:12,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 65 msec 2024-11-16T10:15:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-16T10:15:12,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T10:15:12,385 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:15:12,393 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=251 (was 249) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1174335200_22 at /127.0.0.1:41836 [Waiting for operation #105] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-607888602_22 at /127.0.0.1:41810 [Waiting for operation #160] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1174335200_22 at /127.0.0.1:59204 [Waiting for operation #304] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-607888602_22 at /127.0.0.1:41812 [Waiting for operation #147] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/cluster_bb61cbda-8873-24b8-881e-9662d491e561/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/cluster_bb61cbda-8873-24b8-881e-9662d491e561/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=312 (was 314), ProcessCount=11 (was 11), AvailableMemoryMB=3799 (was 3881) 2024-11-16T10:15:12,403 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=251, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=312, ProcessCount=11, AvailableMemoryMB=3799 2024-11-16T10:15:12,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T10:15:12,405 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T10:15:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T10:15:12,407 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T10:15:12,408 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:12,408 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-16T10:15:12,408 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T10:15:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-16T10:15:12,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742047_1223 (size=1061) 2024-11-16T10:15:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-16T10:15:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-16T10:15:12,821 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0 2024-11-16T10:15:12,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742048_1224 (size=53) 2024-11-16T10:15:13,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-16T10:15:13,231 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:15:13,231 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing dcee08b7bd0f481f30af9d9b29c726c5, disabling compactions & flushes 2024-11-16T10:15:13,231 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:13,231 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:13,231 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. after waiting 0 ms 2024-11-16T10:15:13,231 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:13,231 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:13,231 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for dcee08b7bd0f481f30af9d9b29c726c5: Waiting for close lock at 1731752113231Disabling compacts and flushes for region at 1731752113231Disabling writes for close at 1731752113231Writing region close event to WAL at 1731752113231Closed at 1731752113231 2024-11-16T10:15:13,233 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T10:15:13,233 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731752113233"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731752113233"}]},"ts":"1731752113233"} 2024-11-16T10:15:13,236 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T10:15:13,238 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T10:15:13,238 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752113238"}]},"ts":"1731752113238"} 2024-11-16T10:15:13,240 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T10:15:13,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dcee08b7bd0f481f30af9d9b29c726c5, ASSIGN}] 2024-11-16T10:15:13,242 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dcee08b7bd0f481f30af9d9b29c726c5, ASSIGN 2024-11-16T10:15:13,243 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dcee08b7bd0f481f30af9d9b29c726c5, ASSIGN; state=OFFLINE, location=b16ff38458c1,45417,1731752048912; forceNewPlan=false, retain=false 2024-11-16T10:15:13,394 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=dcee08b7bd0f481f30af9d9b29c726c5, regionState=OPENING, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:13,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dcee08b7bd0f481f30af9d9b29c726c5, ASSIGN because future has completed 2024-11-16T10:15:13,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912}] 2024-11-16T10:15:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-16T10:15:13,554 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:13,554 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:15:13,555 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:13,555 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:15:13,555 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:13,555 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:13,557 INFO [StoreOpener-dcee08b7bd0f481f30af9d9b29c726c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:13,558 INFO [StoreOpener-dcee08b7bd0f481f30af9d9b29c726c5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:15:13,558 INFO [StoreOpener-dcee08b7bd0f481f30af9d9b29c726c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcee08b7bd0f481f30af9d9b29c726c5 columnFamilyName A 2024-11-16T10:15:13,558 DEBUG [StoreOpener-dcee08b7bd0f481f30af9d9b29c726c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:13,559 INFO [StoreOpener-dcee08b7bd0f481f30af9d9b29c726c5-1 {}] regionserver.HStore(327): Store=dcee08b7bd0f481f30af9d9b29c726c5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:15:13,559 INFO [StoreOpener-dcee08b7bd0f481f30af9d9b29c726c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:13,560 INFO [StoreOpener-dcee08b7bd0f481f30af9d9b29c726c5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:15:13,560 INFO [StoreOpener-dcee08b7bd0f481f30af9d9b29c726c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcee08b7bd0f481f30af9d9b29c726c5 columnFamilyName B 2024-11-16T10:15:13,560 DEBUG [StoreOpener-dcee08b7bd0f481f30af9d9b29c726c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:13,561 INFO [StoreOpener-dcee08b7bd0f481f30af9d9b29c726c5-1 {}] regionserver.HStore(327): Store=dcee08b7bd0f481f30af9d9b29c726c5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:15:13,561 INFO [StoreOpener-dcee08b7bd0f481f30af9d9b29c726c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:13,562 INFO [StoreOpener-dcee08b7bd0f481f30af9d9b29c726c5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:15:13,562 INFO [StoreOpener-dcee08b7bd0f481f30af9d9b29c726c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcee08b7bd0f481f30af9d9b29c726c5 columnFamilyName C 2024-11-16T10:15:13,562 DEBUG [StoreOpener-dcee08b7bd0f481f30af9d9b29c726c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:13,563 INFO [StoreOpener-dcee08b7bd0f481f30af9d9b29c726c5-1 {}] regionserver.HStore(327): Store=dcee08b7bd0f481f30af9d9b29c726c5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:15:13,563 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:13,563 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:13,564 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:13,564 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:13,566 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:13,566 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:13,566 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:15:13,568 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:13,570 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T10:15:13,570 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened dcee08b7bd0f481f30af9d9b29c726c5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74832574, jitterRate=0.11509224772453308}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:15:13,570 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:13,571 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for dcee08b7bd0f481f30af9d9b29c726c5: Running coprocessor pre-open hook at 1731752113555Writing region info on filesystem at 1731752113555Initializing all the Stores at 1731752113556 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752113556Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752113556Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752113556Cleaning up temporary data from old regions at 1731752113566 (+10 ms)Running coprocessor post-open hooks at 1731752113570 (+4 ms)Region opened successfully at 1731752113571 (+1 ms) 2024-11-16T10:15:13,571 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., pid=65, masterSystemTime=1731752113547 2024-11-16T10:15:13,573 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:13,573 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:13,573 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=dcee08b7bd0f481f30af9d9b29c726c5, regionState=OPEN, openSeqNum=2, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:13,574 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 because future has completed 2024-11-16T10:15:13,577 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-16T10:15:13,577 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 in 180 msec 2024-11-16T10:15:13,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-16T10:15:13,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dcee08b7bd0f481f30af9d9b29c726c5, ASSIGN in 337 msec 2024-11-16T10:15:13,578 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T10:15:13,579 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752113578"}]},"ts":"1731752113578"} 2024-11-16T10:15:13,580 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T10:15:13,580 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T10:15:13,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1750 sec 2024-11-16T10:15:14,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-16T10:15:14,564 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:15:14,566 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e04d2b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,566 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:15:14,566 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:15:14,567 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:15:14,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:15:14,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:15:14,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52272536, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:15:14,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:15:14,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:14,570 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:15:14,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4905445e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,571 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:15:14,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:15:14,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:15:14,574 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:15:14,576 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T10:15:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is b16ff38458c1,45949,1731752047888 2024-11-16T10:15:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@37284d9b 2024-11-16T10:15:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T10:15:14,579 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T10:15:14,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391cff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,582 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:15:14,582 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:15:14,583 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:15:14,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:15:14,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:15:14,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7557a235, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:15:14,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:15:14,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:14,584 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:15:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bdeac14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,586 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:15:14,586 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:15:14,587 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:15:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:15:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:15:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3360de83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:15:14,588 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:15:14,588 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:14,589 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:15:14,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@254a1c7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,590 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,590 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:15:14,590 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:15:14,591 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:15:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:15:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:15:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4741fa68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:15:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:15:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:14,592 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:15:14,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f6ba75d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,593 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:15:14,593 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:15:14,594 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:15:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:15:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:15:14,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5736515e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:15:14,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:15:14,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:14,596 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:15:14,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@414b571b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,597 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:15:14,597 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:15:14,598 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:15:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:15:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:15:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29d30296, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:15:14,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:15:14,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:14,600 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:15:14,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11b3d362, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,601 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,601 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:15:14,601 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:15:14,602 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:15:14,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:15:14,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:15:14,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@291f5fb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:15:14,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:15:14,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:14,604 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:15:14,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dafb4fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,605 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,605 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:15:14,605 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:15:14,606 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:15:14,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:15:14,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:15:14,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f85b15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:15:14,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:15:14,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:14,608 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:15:14,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f38d64a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,609 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:15:14,609 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:15:14,610 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:15:14,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:15:14,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:15:14,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@688037bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:15:14,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:15:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:14,611 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:15:14,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6efcaeeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861c2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,620 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:15:14,621 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:15:14,622 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:15:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:15:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:15:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c37c162, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:15:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:15:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:14,624 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:15:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5634100d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3397c776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,626 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:15:14,626 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:15:14,627 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:15:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:15:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:15:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@764c0fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:15:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:15:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:14,629 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:15:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23fd289b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:14,632 DEBUG [Thread-1062 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:15:14,632 DEBUG [Thread-1073 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:15:14,632 DEBUG [Thread-1067 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:15:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:15:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:15:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:15:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:15:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:15:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:15:14,635 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:15:14,635 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:15:14,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:15:14,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:15:14,640 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:15:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:15:14,643 DEBUG [Thread-1065 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:15:14,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:15:14,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:15:14,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:15:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:15:14,649 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:15:14,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-16T10:15:14,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T10:15:14,654 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:15:14,655 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:15:14,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:15:14,668 DEBUG [Thread-1071 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:15:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:15:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:15:14,670 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:15:14,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:15:14,680 DEBUG [Thread-1069 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:15:14,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:15:14,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:15:14,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:15:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:15:14,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:14,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T10:15:14,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:14,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:14,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:14,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:14,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:14,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:14,694 DEBUG [Thread-1056 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:15:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:15:14,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:15:14,697 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:15:14,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:15:14,701 DEBUG [Thread-1054 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:15:14,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:15:14,706 DEBUG [Thread-1054 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:15:14,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:15:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:15:14,720 DEBUG [Thread-1058 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:15:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:15:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:15:14,723 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:15:14,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:15:14,725 DEBUG [Thread-1060 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:15:14,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:15:14,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:15:14,728 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:15:14,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:15:14,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/437b4e9e29b74a4e91eb7bcc9a069077 is 50, key is test_row_0/A:col10/1731752114646/Put/seqid=0 2024-11-16T10:15:14,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752114743, max timestamp=1731752114754Num uniques 859; 2024-11-16T10:15:14,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:15:14,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752114743, max timestamp=1731752114754Num uniques 859; 2024-11-16T10:15:14,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752114743, max timestamp=1731752114754Num uniques 859; 2024-11-16T10:15:14,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:15:14,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:15:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T10:15:14,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742049_1225 (size=12001) 2024-11-16T10:15:14,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/437b4e9e29b74a4e91eb7bcc9a069077 2024-11-16T10:15:14,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731752114789, max timestamp=1731752114799Num uniques 1018; 2024-11-16T10:15:14,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-16T10:15:14,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T10:15:14,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:14,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731752114789, max timestamp=1731752114799Num uniques 1018; 2024-11-16T10:15:14,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:14,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:14,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-16T10:15:14,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:14,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:14,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731752114789, max timestamp=1731752114807Num uniques 1018; 2024-11-16T10:15:14,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-16T10:15:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:14,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/455a0c5270034b9393309a19af9309db is 50, key is test_row_0/B:col10/1731752114646/Put/seqid=0 2024-11-16T10:15:14,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742050_1226 (size=12001) 2024-11-16T10:15:14,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:14,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752174846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:14,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752174849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:14,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731752114841, max timestamp=1731752114861Num uniques 1027; 2024-11-16T10:15:14,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731752114841, max timestamp=1731752114861Num uniques 1077; 2024-11-16T10:15:14,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-16T10:15:14,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-16T10:15:14,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731752114841, max timestamp=1731752114861Num uniques 1009; 2024-11-16T10:15:14,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-11-16T10:15:14,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:14,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752174908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752174909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:14,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752174916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:14,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:14,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:14,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752174964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:14,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752174964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:14,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:14,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:14,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T10:15:14,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:14,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:14,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:14,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:14,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:14,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T10:15:15,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:15,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752175024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:15,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752175024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:15,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:15,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752175024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:15,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T10:15:15,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:15,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:15,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:15,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:15,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:15,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752175174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:15,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:15,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752175174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:15,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:15,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:15,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752175234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752175234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:15,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752175234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:15,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:15,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/455a0c5270034b9393309a19af9309db 2024-11-16T10:15:15,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T10:15:15,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:15,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:15,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:15,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:15,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T10:15:15,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/e0ea2ebf7efe4aa5a442f246ac586947 is 50, key is test_row_0/C:col10/1731752114646/Put/seqid=0 2024-11-16T10:15:15,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742051_1227 (size=12001) 2024-11-16T10:15:15,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/e0ea2ebf7efe4aa5a442f246ac586947 2024-11-16T10:15:15,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/437b4e9e29b74a4e91eb7bcc9a069077 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/437b4e9e29b74a4e91eb7bcc9a069077 2024-11-16T10:15:15,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/437b4e9e29b74a4e91eb7bcc9a069077, entries=150, sequenceid=11, filesize=11.7 K 2024-11-16T10:15:15,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/455a0c5270034b9393309a19af9309db as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/455a0c5270034b9393309a19af9309db 2024-11-16T10:15:15,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/455a0c5270034b9393309a19af9309db, entries=150, sequenceid=11, filesize=11.7 K 2024-11-16T10:15:15,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/e0ea2ebf7efe4aa5a442f246ac586947 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/e0ea2ebf7efe4aa5a442f246ac586947 2024-11-16T10:15:15,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/e0ea2ebf7efe4aa5a442f246ac586947, entries=150, sequenceid=11, filesize=11.7 K 2024-11-16T10:15:15,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for dcee08b7bd0f481f30af9d9b29c726c5 in 657ms, sequenceid=11, compaction requested=false 2024-11-16T10:15:15,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:15,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T10:15:15,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:15,435 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=134.18 KB heapSize=383.30 KB 2024-11-16T10:15:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:15,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/b4c741b3a4184aee937779a40608f357 is 50, key is test_row_0/A:col10/1731752114914/Put/seqid=0 2024-11-16T10:15:15,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742052_1228 (size=12001) 2024-11-16T10:15:15,447 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/b4c741b3a4184aee937779a40608f357 2024-11-16T10:15:15,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/03eecf08b0e64c9d950f23da87559707 is 50, key is test_row_0/B:col10/1731752114914/Put/seqid=0 2024-11-16T10:15:15,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742053_1229 (size=12001) 2024-11-16T10:15:15,467 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/03eecf08b0e64c9d950f23da87559707 2024-11-16T10:15:15,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/f2b0398b2db1480497c0ab6f34204771 is 50, key is test_row_0/C:col10/1731752114914/Put/seqid=0 2024-11-16T10:15:15,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:15,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:15,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742054_1230 (size=12001) 2024-11-16T10:15:15,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:15,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752175502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:15,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:15,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:15,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752175509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:15,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:15,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752175544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752175544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:15,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752175544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:15,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:15,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752175614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:15,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:15,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752175624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:15,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:15,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T10:15:15,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:15,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752175824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:15,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:15,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:15,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752175834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:15,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:15,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:15,901 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/f2b0398b2db1480497c0ab6f34204771 2024-11-16T10:15:15,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/b4c741b3a4184aee937779a40608f357 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/b4c741b3a4184aee937779a40608f357 2024-11-16T10:15:15,915 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/b4c741b3a4184aee937779a40608f357, entries=150, sequenceid=82, filesize=11.7 K 2024-11-16T10:15:15,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/03eecf08b0e64c9d950f23da87559707 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/03eecf08b0e64c9d950f23da87559707 2024-11-16T10:15:15,922 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/03eecf08b0e64c9d950f23da87559707, entries=150, sequenceid=82, filesize=11.7 K 2024-11-16T10:15:15,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/f2b0398b2db1480497c0ab6f34204771 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/f2b0398b2db1480497c0ab6f34204771 2024-11-16T10:15:15,928 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/f2b0398b2db1480497c0ab6f34204771, entries=150, sequenceid=82, filesize=11.7 K 2024-11-16T10:15:15,929 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~383.25 KB/392448, currentSize=46.96 KB/48090 for dcee08b7bd0f481f30af9d9b29c726c5 in 495ms, sequenceid=82, compaction requested=false 2024-11-16T10:15:15,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:15,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:15,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-16T10:15:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-16T10:15:15,933 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-16T10:15:15,933 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2750 sec 2024-11-16T10:15:15,936 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.2860 sec 2024-11-16T10:15:16,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:16,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T10:15:16,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:16,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:16,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:16,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:16,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:16,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:16,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/49ae7ced34c04c319671477089fe8cdc is 50, key is test_row_0/A:col10/1731752115508/Put/seqid=0 2024-11-16T10:15:16,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742055_1231 (size=14341) 2024-11-16T10:15:16,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/49ae7ced34c04c319671477089fe8cdc 2024-11-16T10:15:16,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/8a8833fa522149408d59599923aee088 is 50, key is test_row_0/B:col10/1731752115508/Put/seqid=0 2024-11-16T10:15:16,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742056_1232 (size=12001) 2024-11-16T10:15:16,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/8a8833fa522149408d59599923aee088 2024-11-16T10:15:16,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:16,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752176099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:16,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752116088, max timestamp=1731752116102Num uniques 859; 2024-11-16T10:15:16,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:15:16,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752116088, max timestamp=1731752116102Num uniques 859; 2024-11-16T10:15:16,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:15:16,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752116088, max timestamp=1731752116102Num uniques 859; 2024-11-16T10:15:16,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:15:16,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:16,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/09cc4ebfe7fa4f20b4b118f708e04aac is 50, key is test_row_0/C:col10/1731752115508/Put/seqid=0 2024-11-16T10:15:16,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742057_1233 (size=12001) 2024-11-16T10:15:16,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752176156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:16,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:16,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:16,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752176214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:16,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:16,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752176213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752176214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:16,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752176222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:16,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752176324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:16,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:16,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:16,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752176334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:16,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:16,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752176334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:16,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:16,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:16,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752176344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:16,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752176434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:16,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752176534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:16,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:16,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:16,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752176544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:16,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:16,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752176545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:16,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:16,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:16,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/09cc4ebfe7fa4f20b4b118f708e04aac 2024-11-16T10:15:16,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752176554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:16,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:16,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/49ae7ced34c04c319671477089fe8cdc as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/49ae7ced34c04c319671477089fe8cdc 2024-11-16T10:15:16,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/49ae7ced34c04c319671477089fe8cdc, entries=200, sequenceid=93, filesize=14.0 K 2024-11-16T10:15:16,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/8a8833fa522149408d59599923aee088 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/8a8833fa522149408d59599923aee088 2024-11-16T10:15:16,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/8a8833fa522149408d59599923aee088, entries=150, sequenceid=93, filesize=11.7 K 2024-11-16T10:15:16,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/09cc4ebfe7fa4f20b4b118f708e04aac as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/09cc4ebfe7fa4f20b4b118f708e04aac 2024-11-16T10:15:16,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/09cc4ebfe7fa4f20b4b118f708e04aac, entries=150, sequenceid=93, filesize=11.7 K 2024-11-16T10:15:16,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=136.42 KB/139690 for dcee08b7bd0f481f30af9d9b29c726c5 in 531ms, sequenceid=93, compaction requested=true 2024-11-16T10:15:16,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:16,589 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:16,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:15:16,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:16,589 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:16,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:15:16,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:16,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:15:16,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:16,590 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:16,590 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/A is initiating minor compaction (all files) 2024-11-16T10:15:16,590 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/A in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:16,590 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/437b4e9e29b74a4e91eb7bcc9a069077, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/b4c741b3a4184aee937779a40608f357, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/49ae7ced34c04c319671477089fe8cdc] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=37.4 K 2024-11-16T10:15:16,591 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:16,591 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/B is initiating minor compaction (all files) 2024-11-16T10:15:16,591 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/B in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:16,591 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/455a0c5270034b9393309a19af9309db, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/03eecf08b0e64c9d950f23da87559707, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/8a8833fa522149408d59599923aee088] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=35.2 K 2024-11-16T10:15:16,592 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 437b4e9e29b74a4e91eb7bcc9a069077, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731752114646 2024-11-16T10:15:16,594 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 455a0c5270034b9393309a19af9309db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731752114646 2024-11-16T10:15:16,594 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4c741b3a4184aee937779a40608f357, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731752114907 2024-11-16T10:15:16,595 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 03eecf08b0e64c9d950f23da87559707, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731752114907 2024-11-16T10:15:16,595 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49ae7ced34c04c319671477089fe8cdc, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731752115493 2024-11-16T10:15:16,596 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a8833fa522149408d59599923aee088, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731752115493 2024-11-16T10:15:16,607 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#B#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:16,607 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/b9a511fb68f94a79b6865677396f0e9f is 50, key is test_row_0/B:col10/1731752115508/Put/seqid=0 2024-11-16T10:15:16,615 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#A#compaction#193 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:16,616 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/0dc00fb0eab247059285376cc89fd131 is 50, key is test_row_0/A:col10/1731752115508/Put/seqid=0 2024-11-16T10:15:16,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742059_1235 (size=12104) 2024-11-16T10:15:16,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742058_1234 (size=12104) 2024-11-16T10:15:16,672 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/0dc00fb0eab247059285376cc89fd131 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/0dc00fb0eab247059285376cc89fd131 2024-11-16T10:15:16,678 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/A of dcee08b7bd0f481f30af9d9b29c726c5 into 0dc00fb0eab247059285376cc89fd131(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:16,678 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:16,678 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/A, priority=13, startTime=1731752116589; duration=0sec 2024-11-16T10:15:16,678 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:16,678 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:A 2024-11-16T10:15:16,678 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:16,681 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:16,682 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/C is initiating minor compaction (all files) 2024-11-16T10:15:16,682 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/C in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:16,682 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/e0ea2ebf7efe4aa5a442f246ac586947, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/f2b0398b2db1480497c0ab6f34204771, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/09cc4ebfe7fa4f20b4b118f708e04aac] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=35.2 K 2024-11-16T10:15:16,682 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0ea2ebf7efe4aa5a442f246ac586947, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731752114646 2024-11-16T10:15:16,683 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2b0398b2db1480497c0ab6f34204771, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731752114907 2024-11-16T10:15:16,683 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09cc4ebfe7fa4f20b4b118f708e04aac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731752115493 2024-11-16T10:15:16,707 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#C#compaction#194 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:16,708 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/8e6663d15a63406ea10b7c0c0151fddf is 50, key is test_row_0/C:col10/1731752115508/Put/seqid=0 2024-11-16T10:15:16,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742060_1236 (size=12104) 2024-11-16T10:15:16,738 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/8e6663d15a63406ea10b7c0c0151fddf as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/8e6663d15a63406ea10b7c0c0151fddf 2024-11-16T10:15:16,746 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/C of dcee08b7bd0f481f30af9d9b29c726c5 into 8e6663d15a63406ea10b7c0c0151fddf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:16,746 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:16,746 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/C, priority=13, startTime=1731752116589; duration=0sec 2024-11-16T10:15:16,746 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:16,746 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:C 2024-11-16T10:15:16,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=143.13 KB heapSize=410.69 KB 2024-11-16T10:15:16,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:16,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T10:15:16,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:16,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T10:15:16,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:16,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T10:15:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:16,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/38561af7b7b246c1a5a9e2f150611265 is 50, key is test_row_0/A:col10/1731752116747/Put/seqid=0 2024-11-16T10:15:16,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742061_1237 (size=14441) 2024-11-16T10:15:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T10:15:16,804 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:15:16,806 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:15:16,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752176801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:16,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-16T10:15:16,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:16,809 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:15:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T10:15:16,810 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:15:16,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:15:16,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:16,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752176844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:16,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:16,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752176855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:16,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752176855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:16,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:16,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:16,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752176864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:16,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T10:15:16,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752176914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:16,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T10:15:16,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:16,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:16,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:16,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:16,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:17,072 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/b9a511fb68f94a79b6865677396f0e9f as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/b9a511fb68f94a79b6865677396f0e9f 2024-11-16T10:15:17,078 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/B of dcee08b7bd0f481f30af9d9b29c726c5 into b9a511fb68f94a79b6865677396f0e9f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:17,078 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:17,078 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/B, priority=13, startTime=1731752116589; duration=0sec 2024-11-16T10:15:17,078 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:17,078 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:B 2024-11-16T10:15:17,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T10:15:17,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:17,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:17,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:17,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:17,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:17,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T10:15:17,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752177126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:17,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/38561af7b7b246c1a5a9e2f150611265 2024-11-16T10:15:17,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/a2f42fd7d34f47ecb6c7bf7ef24079da is 50, key is test_row_0/B:col10/1731752116747/Put/seqid=0 2024-11-16T10:15:17,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T10:15:17,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:17,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:17,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:17,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:17,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:17,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742062_1238 (size=12051) 2024-11-16T10:15:17,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/a2f42fd7d34f47ecb6c7bf7ef24079da 2024-11-16T10:15:17,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/19e7fd80b46542858568d2b82ebd7aa6 is 50, key is test_row_0/C:col10/1731752116747/Put/seqid=0 2024-11-16T10:15:17,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742063_1239 (size=12051) 2024-11-16T10:15:17,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752177354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:17,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:17,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:17,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:17,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/19e7fd80b46542858568d2b82ebd7aa6 2024-11-16T10:15:17,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/38561af7b7b246c1a5a9e2f150611265 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/38561af7b7b246c1a5a9e2f150611265 2024-11-16T10:15:17,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/38561af7b7b246c1a5a9e2f150611265, entries=200, sequenceid=133, filesize=14.1 K 2024-11-16T10:15:17,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752177364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:17,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/a2f42fd7d34f47ecb6c7bf7ef24079da as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/a2f42fd7d34f47ecb6c7bf7ef24079da 2024-11-16T10:15:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:17,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:17,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:17,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752177364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:17,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:17,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:17,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:17,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/a2f42fd7d34f47ecb6c7bf7ef24079da, entries=150, sequenceid=133, filesize=11.8 K 2024-11-16T10:15:17,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/19e7fd80b46542858568d2b82ebd7aa6 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/19e7fd80b46542858568d2b82ebd7aa6 2024-11-16T10:15:17,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:17,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752177384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:17,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:17,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:17,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:17,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/19e7fd80b46542858568d2b82ebd7aa6, entries=150, sequenceid=133, filesize=11.8 K 2024-11-16T10:15:17,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~143.13 KB/146560, heapSize ~410.64 KB/420496, currentSize=40.25 KB/41220 for dcee08b7bd0f481f30af9d9b29c726c5 in 644ms, sequenceid=133, compaction requested=false 2024-11-16T10:15:17,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:17,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T10:15:17,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:17,428 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-16T10:15:17,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:17,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:17,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:17,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:17,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:17,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:17,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T10:15:17,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:17,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:17,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/93fe9ba1f87240b3a107839e8850c72a is 50, key is test_row_0/A:col10/1731752116787/Put/seqid=0 2024-11-16T10:15:17,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742064_1240 (size=12151) 2024-11-16T10:15:17,476 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/93fe9ba1f87240b3a107839e8850c72a 2024-11-16T10:15:17,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/43a74bdf91604bd2b741bdf99b28201a is 50, key is test_row_0/B:col10/1731752116787/Put/seqid=0 2024-11-16T10:15:17,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742065_1241 (size=12151) 2024-11-16T10:15:17,529 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/43a74bdf91604bd2b741bdf99b28201a 2024-11-16T10:15:17,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/12075f546ef74fa7813e373417f76e18 is 50, key is test_row_0/C:col10/1731752116787/Put/seqid=0 2024-11-16T10:15:17,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752117440, max timestamp=1731752117555Num uniques 859; 2024-11-16T10:15:17,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752117440, max timestamp=1731752117555Num uniques 859; 2024-11-16T10:15:17,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:15:17,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:15:17,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742066_1242 (size=12151) 2024-11-16T10:15:17,581 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T10:15:17,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:17,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752177641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:17,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:17,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752177754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:17,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:17,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:17,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T10:15:17,960 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/12075f546ef74fa7813e373417f76e18 2024-11-16T10:15:17,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/93fe9ba1f87240b3a107839e8850c72a as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/93fe9ba1f87240b3a107839e8850c72a 2024-11-16T10:15:17,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752177964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:17,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:17,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:17,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:17,977 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/93fe9ba1f87240b3a107839e8850c72a, entries=150, sequenceid=143, filesize=11.9 K 2024-11-16T10:15:17,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/43a74bdf91604bd2b741bdf99b28201a as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/43a74bdf91604bd2b741bdf99b28201a 2024-11-16T10:15:17,991 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/43a74bdf91604bd2b741bdf99b28201a, entries=150, sequenceid=143, filesize=11.9 K 2024-11-16T10:15:17,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/12075f546ef74fa7813e373417f76e18 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/12075f546ef74fa7813e373417f76e18 2024-11-16T10:15:18,001 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/12075f546ef74fa7813e373417f76e18, entries=150, sequenceid=143, filesize=11.9 K 2024-11-16T10:15:18,004 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=138.65 KB/141980 for dcee08b7bd0f481f30af9d9b29c726c5 in 575ms, sequenceid=143, compaction requested=true 2024-11-16T10:15:18,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:18,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:18,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-16T10:15:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-16T10:15:18,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-16T10:15:18,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1950 sec 2024-11-16T10:15:18,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.2030 sec 2024-11-16T10:15:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:18,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=145.36 KB heapSize=423.16 KB 2024-11-16T10:15:18,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:18,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:18,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:18,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:18,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:18,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:18,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/da21fe390bc44a84ac89b8a74eca2f70 is 50, key is test_row_0/A:col10/1731752118276/Put/seqid=0 2024-11-16T10:15:18,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742067_1243 (size=14541) 2024-11-16T10:15:18,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752178311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:18,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:18,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:18,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:18,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752178374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:18,380 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_0' in dcee08b7bd0f481f30af9d9b29c726c5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:15:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:18,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752178384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:18,389 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in dcee08b7bd0f481f30af9d9b29c726c5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:15:18,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:18,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:18,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:18,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:18,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752178394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:18,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752178394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:18,399 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_0' in dcee08b7bd0f481f30af9d9b29c726c5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:15:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:18,400 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_0' in dcee08b7bd0f481f30af9d9b29c726c5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:15:18,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:18,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:18,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:18,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:18,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752178424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:18,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:18,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:18,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752178634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:18,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:18,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:18,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:18,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/da21fe390bc44a84ac89b8a74eca2f70 2024-11-16T10:15:18,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/b4867d784dce44c9a6253ed3dbd31476 is 50, key is test_row_0/B:col10/1731752118276/Put/seqid=0 2024-11-16T10:15:18,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742068_1244 (size=12151) 2024-11-16T10:15:18,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752178944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T10:15:18,954 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:15:18,955 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:15:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-16T10:15:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T10:15:18,956 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:15:18,957 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:15:18,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:15:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T10:15:19,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T10:15:19,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:19,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:19,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:19,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:19,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:19,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/b4867d784dce44c9a6253ed3dbd31476 2024-11-16T10:15:19,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/8bdb1aa1e3da4834b60d738b0588b49c is 50, key is test_row_0/C:col10/1731752118276/Put/seqid=0 2024-11-16T10:15:19,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742069_1245 (size=12151) 2024-11-16T10:15:19,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T10:15:19,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:19,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:19,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:19,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:19,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T10:15:19,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T10:15:19,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:19,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:19,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:19,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:19,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:19,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752179464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:19,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:19,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:19,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:19,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=69.33 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/8bdb1aa1e3da4834b60d738b0588b49c 2024-11-16T10:15:19,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/da21fe390bc44a84ac89b8a74eca2f70 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/da21fe390bc44a84ac89b8a74eca2f70 2024-11-16T10:15:19,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/da21fe390bc44a84ac89b8a74eca2f70, entries=200, sequenceid=177, filesize=14.2 K 2024-11-16T10:15:19,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/b4867d784dce44c9a6253ed3dbd31476 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/b4867d784dce44c9a6253ed3dbd31476 2024-11-16T10:15:19,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/b4867d784dce44c9a6253ed3dbd31476, entries=150, sequenceid=177, filesize=11.9 K 2024-11-16T10:15:19,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/8bdb1aa1e3da4834b60d738b0588b49c as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/8bdb1aa1e3da4834b60d738b0588b49c 2024-11-16T10:15:19,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/8bdb1aa1e3da4834b60d738b0588b49c, entries=150, sequenceid=177, filesize=11.9 K 2024-11-16T10:15:19,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~145.36 KB/148850, heapSize ~423.11 KB/433264, currentSize=33.54 KB/34350 for dcee08b7bd0f481f30af9d9b29c726c5 in 1290ms, sequenceid=177, compaction requested=true 2024-11-16T10:15:19,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:19,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T10:15:19,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:19,569 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:15:19,569 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-16T10:15:19,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:15:19,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:19,569 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:15:19,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:19,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:15:19,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:19,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:19,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:19,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:19,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:19,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:15:19,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:19,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:19,570 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53237 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:15:19,571 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/A is initiating minor compaction (all files) 2024-11-16T10:15:19,571 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48457 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:15:19,571 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/A in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:19,571 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/B is initiating minor compaction (all files) 2024-11-16T10:15:19,571 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/B in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:19,571 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/0dc00fb0eab247059285376cc89fd131, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/38561af7b7b246c1a5a9e2f150611265, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/93fe9ba1f87240b3a107839e8850c72a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/da21fe390bc44a84ac89b8a74eca2f70] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=52.0 K 2024-11-16T10:15:19,571 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/b9a511fb68f94a79b6865677396f0e9f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/a2f42fd7d34f47ecb6c7bf7ef24079da, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/43a74bdf91604bd2b741bdf99b28201a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/b4867d784dce44c9a6253ed3dbd31476] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=47.3 K 2024-11-16T10:15:19,571 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dc00fb0eab247059285376cc89fd131, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731752115493 2024-11-16T10:15:19,571 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting b9a511fb68f94a79b6865677396f0e9f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731752115493 2024-11-16T10:15:19,572 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38561af7b7b246c1a5a9e2f150611265, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731752116133 2024-11-16T10:15:19,572 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting a2f42fd7d34f47ecb6c7bf7ef24079da, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731752116133 2024-11-16T10:15:19,572 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93fe9ba1f87240b3a107839e8850c72a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731752116781 2024-11-16T10:15:19,572 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 43a74bdf91604bd2b741bdf99b28201a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731752116781 2024-11-16T10:15:19,573 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting b4867d784dce44c9a6253ed3dbd31476, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731752117618 2024-11-16T10:15:19,573 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting da21fe390bc44a84ac89b8a74eca2f70, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731752117618 2024-11-16T10:15:19,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/627750da63b842bc8d77d790dc877409 is 50, key is test_row_0/A:col10/1731752118294/Put/seqid=0 2024-11-16T10:15:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T10:15:19,592 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#B#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:19,593 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/d8fea82f00e44ff0a666d1b5a3145d54 is 50, key is test_row_0/B:col10/1731752118276/Put/seqid=0 2024-11-16T10:15:19,603 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#A#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:19,604 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/de5dfec7b96946ef80ec5950323f4a19 is 50, key is test_row_0/A:col10/1731752118276/Put/seqid=0 2024-11-16T10:15:19,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742070_1246 (size=12151) 2024-11-16T10:15:19,650 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/627750da63b842bc8d77d790dc877409 2024-11-16T10:15:19,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742071_1247 (size=12391) 2024-11-16T10:15:19,680 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/d8fea82f00e44ff0a666d1b5a3145d54 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/d8fea82f00e44ff0a666d1b5a3145d54 2024-11-16T10:15:19,693 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/B of dcee08b7bd0f481f30af9d9b29c726c5 into d8fea82f00e44ff0a666d1b5a3145d54(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:19,693 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:19,693 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/B, priority=12, startTime=1731752119569; duration=0sec 2024-11-16T10:15:19,693 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:19,694 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:B 2024-11-16T10:15:19,694 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:15:19,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742072_1248 (size=12391) 2024-11-16T10:15:19,696 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48457 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:15:19,696 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/C is initiating minor compaction (all files) 2024-11-16T10:15:19,696 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/C in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:19,696 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/8e6663d15a63406ea10b7c0c0151fddf, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/19e7fd80b46542858568d2b82ebd7aa6, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/12075f546ef74fa7813e373417f76e18, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/8bdb1aa1e3da4834b60d738b0588b49c] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=47.3 K 2024-11-16T10:15:19,697 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e6663d15a63406ea10b7c0c0151fddf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731752115493 2024-11-16T10:15:19,698 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 19e7fd80b46542858568d2b82ebd7aa6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731752116133 2024-11-16T10:15:19,699 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 12075f546ef74fa7813e373417f76e18, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731752116781 2024-11-16T10:15:19,700 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bdb1aa1e3da4834b60d738b0588b49c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731752117618 2024-11-16T10:15:19,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/d9a98c27e5844a958f3cbe78e5a922d2 is 50, key is test_row_0/B:col10/1731752118294/Put/seqid=0 2024-11-16T10:15:19,702 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/de5dfec7b96946ef80ec5950323f4a19 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/de5dfec7b96946ef80ec5950323f4a19 2024-11-16T10:15:19,709 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/A of dcee08b7bd0f481f30af9d9b29c726c5 into de5dfec7b96946ef80ec5950323f4a19(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:19,709 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:19,709 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/A, priority=12, startTime=1731752119569; duration=0sec 2024-11-16T10:15:19,711 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:19,711 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:A 2024-11-16T10:15:19,721 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#C#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:19,722 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/6f04a8393324424283f8f22f689f55f8 is 50, key is test_row_0/C:col10/1731752118276/Put/seqid=0 2024-11-16T10:15:19,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742073_1249 (size=12151) 2024-11-16T10:15:19,746 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/d9a98c27e5844a958f3cbe78e5a922d2 2024-11-16T10:15:19,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/bc1ba5f7a0064d0281c07234c95cd3cd is 50, key is test_row_0/C:col10/1731752118294/Put/seqid=0 2024-11-16T10:15:19,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742074_1250 (size=12391) 2024-11-16T10:15:19,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742075_1251 (size=12151) 2024-11-16T10:15:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T10:15:20,218 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/bc1ba5f7a0064d0281c07234c95cd3cd 2024-11-16T10:15:20,225 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/6f04a8393324424283f8f22f689f55f8 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/6f04a8393324424283f8f22f689f55f8 2024-11-16T10:15:20,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/627750da63b842bc8d77d790dc877409 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/627750da63b842bc8d77d790dc877409 2024-11-16T10:15:20,231 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/627750da63b842bc8d77d790dc877409, entries=150, sequenceid=185, filesize=11.9 K 2024-11-16T10:15:20,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/d9a98c27e5844a958f3cbe78e5a922d2 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/d9a98c27e5844a958f3cbe78e5a922d2 2024-11-16T10:15:20,232 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/C of dcee08b7bd0f481f30af9d9b29c726c5 into 6f04a8393324424283f8f22f689f55f8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:20,232 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:20,233 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/C, priority=12, startTime=1731752119569; duration=0sec 2024-11-16T10:15:20,233 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:20,233 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:C 2024-11-16T10:15:20,236 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/d9a98c27e5844a958f3cbe78e5a922d2, entries=150, sequenceid=185, filesize=11.9 K 2024-11-16T10:15:20,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/bc1ba5f7a0064d0281c07234c95cd3cd as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/bc1ba5f7a0064d0281c07234c95cd3cd 2024-11-16T10:15:20,244 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/bc1ba5f7a0064d0281c07234c95cd3cd, entries=150, sequenceid=185, filesize=11.9 K 2024-11-16T10:15:20,245 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for dcee08b7bd0f481f30af9d9b29c726c5 in 676ms, sequenceid=185, compaction requested=false 2024-11-16T10:15:20,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:20,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:20,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-16T10:15:20,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-16T10:15:20,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-16T10:15:20,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2890 sec 2024-11-16T10:15:20,252 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.2960 sec 2024-11-16T10:15:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:20,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T10:15:20,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:20,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:20,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:20,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:20,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:20,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:20,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/859b53a043b14ab4b43d0dfa75d6d1c5 is 50, key is test_row_0/A:col10/1731752120441/Put/seqid=0 2024-11-16T10:15:20,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742076_1252 (size=14541) 2024-11-16T10:15:20,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752120459, max timestamp=1731752120468Num uniques 859; 2024-11-16T10:15:20,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:15:20,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752120459, max timestamp=1731752120468Num uniques 859; 2024-11-16T10:15:20,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752120459, max timestamp=1731752120468Num uniques 859; 2024-11-16T10:15:20,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:15:20,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:15:20,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752180488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:20,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:20,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752180489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:20,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:20,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:20,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752180490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752180491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:20,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:20,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752180492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:20,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:20,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:20,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:20,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:20,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752180604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:20,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752180604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:20,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:20,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752180604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:20,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752180604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:20,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752180604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:20,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:20,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752180814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:20,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:20,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:20,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752180814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:20,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:20,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:20,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752180814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:20,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:20,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:20,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752180814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:20,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:20,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:20,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752180814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:20,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:20,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:20,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/859b53a043b14ab4b43d0dfa75d6d1c5 2024-11-16T10:15:20,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/bc535bea1553454bafe3248b8b484457 is 50, key is test_row_0/B:col10/1731752120441/Put/seqid=0 2024-11-16T10:15:20,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742077_1253 (size=12151) 2024-11-16T10:15:20,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/bc535bea1553454bafe3248b8b484457 2024-11-16T10:15:20,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/98e94354a0c34e19976b8e00dd0a8a44 is 50, key is test_row_0/C:col10/1731752120441/Put/seqid=0 2024-11-16T10:15:20,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742078_1254 (size=12151) 2024-11-16T10:15:20,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/98e94354a0c34e19976b8e00dd0a8a44 2024-11-16T10:15:20,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/859b53a043b14ab4b43d0dfa75d6d1c5 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/859b53a043b14ab4b43d0dfa75d6d1c5 2024-11-16T10:15:20,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/859b53a043b14ab4b43d0dfa75d6d1c5, entries=200, sequenceid=198, filesize=14.2 K 2024-11-16T10:15:20,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/bc535bea1553454bafe3248b8b484457 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/bc535bea1553454bafe3248b8b484457 2024-11-16T10:15:20,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/bc535bea1553454bafe3248b8b484457, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T10:15:20,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/98e94354a0c34e19976b8e00dd0a8a44 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/98e94354a0c34e19976b8e00dd0a8a44 2024-11-16T10:15:20,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/98e94354a0c34e19976b8e00dd0a8a44, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T10:15:20,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for dcee08b7bd0f481f30af9d9b29c726c5 in 475ms, sequenceid=198, compaction requested=true 2024-11-16T10:15:20,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:20,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:15:20,918 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:20,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:20,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:15:20,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:20,918 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:20,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:15:20,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:20,919 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39083 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:20,919 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/A is initiating minor compaction (all files) 2024-11-16T10:15:20,919 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/A in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:20,919 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/de5dfec7b96946ef80ec5950323f4a19, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/627750da63b842bc8d77d790dc877409, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/859b53a043b14ab4b43d0dfa75d6d1c5] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=38.2 K 2024-11-16T10:15:20,920 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:20,920 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/B is initiating minor compaction (all files) 2024-11-16T10:15:20,920 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/B in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:20,920 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/d8fea82f00e44ff0a666d1b5a3145d54, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/d9a98c27e5844a958f3cbe78e5a922d2, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/bc535bea1553454bafe3248b8b484457] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=35.8 K 2024-11-16T10:15:20,920 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting de5dfec7b96946ef80ec5950323f4a19, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731752117618 2024-11-16T10:15:20,921 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting d8fea82f00e44ff0a666d1b5a3145d54, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731752117618 2024-11-16T10:15:20,921 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 627750da63b842bc8d77d790dc877409, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731752118294 2024-11-16T10:15:20,921 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting d9a98c27e5844a958f3cbe78e5a922d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731752118294 2024-11-16T10:15:20,921 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 859b53a043b14ab4b43d0dfa75d6d1c5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731752120440 2024-11-16T10:15:20,922 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting bc535bea1553454bafe3248b8b484457, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731752120441 2024-11-16T10:15:20,938 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#B#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:20,938 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#A#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:20,939 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/c36586498d5341949c635c911ca87c21 is 50, key is test_row_0/A:col10/1731752120441/Put/seqid=0 2024-11-16T10:15:20,939 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/ec7a274901354bad8e00fb71c6866c9d is 50, key is test_row_0/B:col10/1731752120441/Put/seqid=0 2024-11-16T10:15:20,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742079_1255 (size=12493) 2024-11-16T10:15:20,962 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/c36586498d5341949c635c911ca87c21 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/c36586498d5341949c635c911ca87c21 2024-11-16T10:15:20,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742080_1256 (size=12493) 2024-11-16T10:15:20,974 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/A of dcee08b7bd0f481f30af9d9b29c726c5 into c36586498d5341949c635c911ca87c21(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:20,974 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:20,974 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/A, priority=13, startTime=1731752120917; duration=0sec 2024-11-16T10:15:20,975 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:20,975 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:A 2024-11-16T10:15:20,975 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:20,976 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:20,976 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/C is initiating minor compaction (all files) 2024-11-16T10:15:20,976 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/C in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:20,976 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/6f04a8393324424283f8f22f689f55f8, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/bc1ba5f7a0064d0281c07234c95cd3cd, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/98e94354a0c34e19976b8e00dd0a8a44] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=35.8 K 2024-11-16T10:15:20,976 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f04a8393324424283f8f22f689f55f8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731752117618 2024-11-16T10:15:20,977 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc1ba5f7a0064d0281c07234c95cd3cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731752118294 2024-11-16T10:15:20,977 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98e94354a0c34e19976b8e00dd0a8a44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731752120441 2024-11-16T10:15:20,988 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#C#compaction#215 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:20,988 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/b9b537acd31543d4b8d776883b8063b4 is 50, key is test_row_0/C:col10/1731752120441/Put/seqid=0 2024-11-16T10:15:21,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742081_1257 (size=12493) 2024-11-16T10:15:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T10:15:21,105 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:15:21,105 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:15:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-16T10:15:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T10:15:21,109 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:15:21,110 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:15:21,110 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:15:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:21,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-16T10:15:21,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7681, totalHeapSize=17712, min timestamp=1731752120483, max timestamp=1731752121128Num uniques 1068; 2024-11-16T10:15:21,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48901, new segment data size=7681, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-16T10:15:21,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:21,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:21,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:21,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T10:15:21,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:21,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T10:15:21,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/89ea2b1096334e4e9f35badfb96bcb45 is 50, key is test_row_0/A:col10/1731752120491/Put/seqid=0 2024-11-16T10:15:21,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742082_1258 (size=12151) 2024-11-16T10:15:21,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752181143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752181143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752181144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752181151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752181152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T10:15:21,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T10:15:21,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:21,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:21,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:21,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752181255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:21,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:21,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752181255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:21,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752181264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752181264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752181264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,379 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/ec7a274901354bad8e00fb71c6866c9d as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/ec7a274901354bad8e00fb71c6866c9d 2024-11-16T10:15:21,387 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/B of dcee08b7bd0f481f30af9d9b29c726c5 into ec7a274901354bad8e00fb71c6866c9d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:21,387 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:21,387 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/B, priority=13, startTime=1731752120918; duration=0sec 2024-11-16T10:15:21,388 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:21,388 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:B 2024-11-16T10:15:21,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T10:15:21,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:21,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:21,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:21,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:21,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:21,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T10:15:21,453 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/b9b537acd31543d4b8d776883b8063b4 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/b9b537acd31543d4b8d776883b8063b4 2024-11-16T10:15:21,461 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/C of dcee08b7bd0f481f30af9d9b29c726c5 into b9b537acd31543d4b8d776883b8063b4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:21,461 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:21,461 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/C, priority=13, startTime=1731752120918; duration=0sec 2024-11-16T10:15:21,461 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:21,461 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:C 2024-11-16T10:15:21,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752181475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752181475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752181475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752181485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752181485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/89ea2b1096334e4e9f35badfb96bcb45 2024-11-16T10:15:21,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/d86f79cdee2e45d593ea7c25948ede97 is 50, key is test_row_0/B:col10/1731752120491/Put/seqid=0 2024-11-16T10:15:21,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T10:15:21,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:21,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:21,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:21,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:21,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:21,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742083_1259 (size=12151) 2024-11-16T10:15:21,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/d86f79cdee2e45d593ea7c25948ede97 2024-11-16T10:15:21,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/d5b766d686c14dc2bb93aee175b2b124 is 50, key is test_row_0/C:col10/1731752120491/Put/seqid=0 2024-11-16T10:15:21,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742084_1260 (size=12151) 2024-11-16T10:15:21,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/d5b766d686c14dc2bb93aee175b2b124 2024-11-16T10:15:21,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/89ea2b1096334e4e9f35badfb96bcb45 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/89ea2b1096334e4e9f35badfb96bcb45 2024-11-16T10:15:21,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/89ea2b1096334e4e9f35badfb96bcb45, entries=150, sequenceid=238, filesize=11.9 K 2024-11-16T10:15:21,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/d86f79cdee2e45d593ea7c25948ede97 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/d86f79cdee2e45d593ea7c25948ede97 2024-11-16T10:15:21,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/d86f79cdee2e45d593ea7c25948ede97, entries=150, sequenceid=238, filesize=11.9 K 2024-11-16T10:15:21,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/d5b766d686c14dc2bb93aee175b2b124 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/d5b766d686c14dc2bb93aee175b2b124 2024-11-16T10:15:21,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/d5b766d686c14dc2bb93aee175b2b124, entries=150, sequenceid=238, filesize=11.9 K 2024-11-16T10:15:21,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~322.23 KB/329968, currentSize=67.09 KB/68700 for dcee08b7bd0f481f30af9d9b29c726c5 in 573ms, sequenceid=238, compaction requested=false 2024-11-16T10:15:21,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:21,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T10:15:21,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:21,726 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T10:15:21,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:21,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:21,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:21,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:21,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:21,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T10:15:21,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/3135512089f94d1b8ce57a0a9c1cd66d is 50, key is test_row_0/A:col10/1731752121140/Put/seqid=0 2024-11-16T10:15:21,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742085_1261 (size=12151) 2024-11-16T10:15:21,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:21,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:21,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752181807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752181809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752181810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752181811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752181812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752181924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752181924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752181924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752181924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752181924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:22,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752182134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:22,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:22,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752182134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:22,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752182134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752182134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:22,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752182134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:22,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:22,162 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/3135512089f94d1b8ce57a0a9c1cd66d 2024-11-16T10:15:22,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/89575ef3838d495998d8494c7432b7ef is 50, key is test_row_0/B:col10/1731752121140/Put/seqid=0 2024-11-16T10:15:22,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742086_1262 (size=12151) 2024-11-16T10:15:22,177 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/89575ef3838d495998d8494c7432b7ef 2024-11-16T10:15:22,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/fd256d2f236f4b0d9427be71a8cd9af3 is 50, key is test_row_0/C:col10/1731752121140/Put/seqid=0 2024-11-16T10:15:22,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742087_1263 (size=12151) 2024-11-16T10:15:22,203 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/fd256d2f236f4b0d9427be71a8cd9af3 2024-11-16T10:15:22,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/3135512089f94d1b8ce57a0a9c1cd66d as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/3135512089f94d1b8ce57a0a9c1cd66d 2024-11-16T10:15:22,222 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/3135512089f94d1b8ce57a0a9c1cd66d, entries=150, sequenceid=253, filesize=11.9 K 2024-11-16T10:15:22,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/89575ef3838d495998d8494c7432b7ef as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/89575ef3838d495998d8494c7432b7ef 2024-11-16T10:15:22,232 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/89575ef3838d495998d8494c7432b7ef, entries=150, sequenceid=253, filesize=11.9 K 2024-11-16T10:15:22,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/fd256d2f236f4b0d9427be71a8cd9af3 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/fd256d2f236f4b0d9427be71a8cd9af3 2024-11-16T10:15:22,240 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/fd256d2f236f4b0d9427be71a8cd9af3, entries=150, sequenceid=253, filesize=11.9 K 2024-11-16T10:15:22,243 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for dcee08b7bd0f481f30af9d9b29c726c5 in 517ms, sequenceid=253, compaction requested=true 2024-11-16T10:15:22,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:22,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:22,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-16T10:15:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T10:15:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-16T10:15:22,247 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-16T10:15:22,247 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1350 sec 2024-11-16T10:15:22,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.1430 sec 2024-11-16T10:15:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:22,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T10:15:22,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:22,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:22,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:22,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:22,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:22,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:22,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/807b40800ec34867a9bb67f14a392b6e is 50, key is test_row_0/A:col10/1731752121809/Put/seqid=0 2024-11-16T10:15:22,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752182455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:22,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:22,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752182457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:22,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:22,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752182459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752182459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752182459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:22,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742088_1264 (size=12301) 2024-11-16T10:15:22,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752182564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:22,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752182574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:22,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752182574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752182574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:22,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752182575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:22,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:22,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752182774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:22,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:22,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752182784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:22,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:22,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752182784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:22,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:22,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752182784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:22,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752182784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:22,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:22,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/807b40800ec34867a9bb67f14a392b6e 2024-11-16T10:15:22,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/d4e60aae835b4d42bb21a2f5fad19274 is 50, key is test_row_0/B:col10/1731752121809/Put/seqid=0 2024-11-16T10:15:22,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742089_1265 (size=12301) 2024-11-16T10:15:23,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752183084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:23,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752183094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:23,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:23,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752183094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:23,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:23,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752183094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:23,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:23,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752183094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:23,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T10:15:23,255 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:15:23,255 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:15:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-16T10:15:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T10:15:23,257 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:15:23,257 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:15:23,257 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:15:23,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/d4e60aae835b4d42bb21a2f5fad19274 2024-11-16T10:15:23,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/30ea12433f68464da3af7b8f2090a490 is 50, key is test_row_0/C:col10/1731752121809/Put/seqid=0 2024-11-16T10:15:23,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742090_1266 (size=12301) 2024-11-16T10:15:23,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/30ea12433f68464da3af7b8f2090a490 2024-11-16T10:15:23,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/807b40800ec34867a9bb67f14a392b6e as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/807b40800ec34867a9bb67f14a392b6e 2024-11-16T10:15:23,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/807b40800ec34867a9bb67f14a392b6e, entries=150, sequenceid=274, filesize=12.0 K 2024-11-16T10:15:23,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/d4e60aae835b4d42bb21a2f5fad19274 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/d4e60aae835b4d42bb21a2f5fad19274 2024-11-16T10:15:23,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/d4e60aae835b4d42bb21a2f5fad19274, entries=150, sequenceid=274, filesize=12.0 K 2024-11-16T10:15:23,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/30ea12433f68464da3af7b8f2090a490 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/30ea12433f68464da3af7b8f2090a490 2024-11-16T10:15:23,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/30ea12433f68464da3af7b8f2090a490, entries=150, sequenceid=274, filesize=12.0 K 2024-11-16T10:15:23,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for dcee08b7bd0f481f30af9d9b29c726c5 in 872ms, sequenceid=274, compaction requested=true 2024-11-16T10:15:23,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:23,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:15:23,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:23,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:15:23,320 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:15:23,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:23,320 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:15:23,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:15:23,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:23,321 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49096 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:15:23,321 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/B is initiating minor compaction (all files) 2024-11-16T10:15:23,321 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/B in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:23,321 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/ec7a274901354bad8e00fb71c6866c9d, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/d86f79cdee2e45d593ea7c25948ede97, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/89575ef3838d495998d8494c7432b7ef, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/d4e60aae835b4d42bb21a2f5fad19274] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=47.9 K 2024-11-16T10:15:23,322 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49096 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:15:23,322 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting ec7a274901354bad8e00fb71c6866c9d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731752120441 2024-11-16T10:15:23,322 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/A is initiating minor compaction (all files) 2024-11-16T10:15:23,322 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/A in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:23,323 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/c36586498d5341949c635c911ca87c21, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/89ea2b1096334e4e9f35badfb96bcb45, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/3135512089f94d1b8ce57a0a9c1cd66d, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/807b40800ec34867a9bb67f14a392b6e] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=47.9 K 2024-11-16T10:15:23,323 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting d86f79cdee2e45d593ea7c25948ede97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731752120491 2024-11-16T10:15:23,323 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting c36586498d5341949c635c911ca87c21, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731752120441 2024-11-16T10:15:23,323 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 89575ef3838d495998d8494c7432b7ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731752121140 2024-11-16T10:15:23,323 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89ea2b1096334e4e9f35badfb96bcb45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731752120491 2024-11-16T10:15:23,323 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting d4e60aae835b4d42bb21a2f5fad19274, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731752121809 2024-11-16T10:15:23,324 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3135512089f94d1b8ce57a0a9c1cd66d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731752121140 2024-11-16T10:15:23,324 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 807b40800ec34867a9bb67f14a392b6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731752121809 2024-11-16T10:15:23,338 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#A#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:23,339 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/ca9905a026fb4e18a97444f87d9a0870 is 50, key is test_row_0/A:col10/1731752121809/Put/seqid=0 2024-11-16T10:15:23,339 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#B#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:23,340 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/64c948eaee2a4a50b83a1c4af2efc6a3 is 50, key is test_row_0/B:col10/1731752121809/Put/seqid=0 2024-11-16T10:15:23,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742091_1267 (size=12779) 2024-11-16T10:15:23,361 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/ca9905a026fb4e18a97444f87d9a0870 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/ca9905a026fb4e18a97444f87d9a0870 2024-11-16T10:15:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T10:15:23,370 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/A of dcee08b7bd0f481f30af9d9b29c726c5 into ca9905a026fb4e18a97444f87d9a0870(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:23,370 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:23,370 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/A, priority=12, startTime=1731752123319; duration=0sec 2024-11-16T10:15:23,370 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:23,370 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:A 2024-11-16T10:15:23,370 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:15:23,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742092_1268 (size=12779) 2024-11-16T10:15:23,373 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49096 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:15:23,373 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/C is initiating minor compaction (all files) 2024-11-16T10:15:23,373 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/C in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:23,373 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/b9b537acd31543d4b8d776883b8063b4, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/d5b766d686c14dc2bb93aee175b2b124, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/fd256d2f236f4b0d9427be71a8cd9af3, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/30ea12433f68464da3af7b8f2090a490] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=47.9 K 2024-11-16T10:15:23,374 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9b537acd31543d4b8d776883b8063b4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731752120441 2024-11-16T10:15:23,375 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5b766d686c14dc2bb93aee175b2b124, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731752120491 2024-11-16T10:15:23,375 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd256d2f236f4b0d9427be71a8cd9af3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731752121140 2024-11-16T10:15:23,376 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30ea12433f68464da3af7b8f2090a490, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731752121809 2024-11-16T10:15:23,377 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/64c948eaee2a4a50b83a1c4af2efc6a3 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/64c948eaee2a4a50b83a1c4af2efc6a3 2024-11-16T10:15:23,384 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/B of dcee08b7bd0f481f30af9d9b29c726c5 into 64c948eaee2a4a50b83a1c4af2efc6a3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:23,384 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:23,384 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/B, priority=12, startTime=1731752123319; duration=0sec 2024-11-16T10:15:23,384 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:23,384 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:B 2024-11-16T10:15:23,390 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#C#compaction#227 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:23,390 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/1e0d9e221d68437e8a2298c0aa743675 is 50, key is test_row_0/C:col10/1731752121809/Put/seqid=0 2024-11-16T10:15:23,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742093_1269 (size=12779) 2024-11-16T10:15:23,400 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/1e0d9e221d68437e8a2298c0aa743675 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/1e0d9e221d68437e8a2298c0aa743675 2024-11-16T10:15:23,405 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/C of dcee08b7bd0f481f30af9d9b29c726c5 into 1e0d9e221d68437e8a2298c0aa743675(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:23,405 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:23,405 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/C, priority=12, startTime=1731752123320; duration=0sec 2024-11-16T10:15:23,405 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:23,405 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:C 2024-11-16T10:15:23,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T10:15:23,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:23,410 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T10:15:23,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:23,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:23,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:23,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:23,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:23,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:23,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/5e8673ca927e4cb993309134e4958043 is 50, key is test_row_0/A:col10/1731752122449/Put/seqid=0 2024-11-16T10:15:23,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742094_1270 (size=12301) 2024-11-16T10:15:23,431 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/5e8673ca927e4cb993309134e4958043 2024-11-16T10:15:23,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/23666c9856494979b4ede6dba94febae is 50, key is test_row_0/B:col10/1731752122449/Put/seqid=0 2024-11-16T10:15:23,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742095_1271 (size=12301) 2024-11-16T10:15:23,456 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/23666c9856494979b4ede6dba94febae 2024-11-16T10:15:23,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/1c41b19cc02f403d9e748731ee1597df is 50, key is test_row_0/C:col10/1731752122449/Put/seqid=0 2024-11-16T10:15:23,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742096_1272 (size=12301) 2024-11-16T10:15:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T10:15:23,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:23,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:23,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752183615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:23,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752183616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:23,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:23,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731752123612, max timestamp=1731752123619Num uniques 859; 2024-11-16T10:15:23,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731752123612, max timestamp=1731752123619Num uniques 859; 2024-11-16T10:15:23,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T10:15:23,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T10:15:23,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:23,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731752123612, max timestamp=1731752123619Num uniques 859; 2024-11-16T10:15:23,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T10:15:23,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752183650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:23,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:23,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752183651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752183652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:23,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752183724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:23,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752183734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:23,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752183764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:23,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:23,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752183764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:23,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:23,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:23,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752183765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:23,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T10:15:23,895 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/1c41b19cc02f403d9e748731ee1597df 2024-11-16T10:15:23,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/5e8673ca927e4cb993309134e4958043 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/5e8673ca927e4cb993309134e4958043 2024-11-16T10:15:23,905 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/5e8673ca927e4cb993309134e4958043, entries=150, sequenceid=288, filesize=12.0 K 2024-11-16T10:15:23,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/23666c9856494979b4ede6dba94febae as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/23666c9856494979b4ede6dba94febae 2024-11-16T10:15:23,911 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/23666c9856494979b4ede6dba94febae, entries=150, sequenceid=288, filesize=12.0 K 2024-11-16T10:15:23,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/1c41b19cc02f403d9e748731ee1597df as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/1c41b19cc02f403d9e748731ee1597df 2024-11-16T10:15:23,916 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/1c41b19cc02f403d9e748731ee1597df, entries=150, sequenceid=288, filesize=12.0 K 2024-11-16T10:15:23,917 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for dcee08b7bd0f481f30af9d9b29c726c5 in 507ms, sequenceid=288, compaction requested=false 2024-11-16T10:15:23,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:23,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:23,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-16T10:15:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-16T10:15:23,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-16T10:15:23,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 661 msec 2024-11-16T10:15:23,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 666 msec 2024-11-16T10:15:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:23,939 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-16T10:15:23,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:23,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:23,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:23,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:23,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:23,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:23,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/febad6b4d1e0468d85c1e74f1b78d49b is 50, key is test_row_0/A:col10/1731752123651/Put/seqid=0 2024-11-16T10:15:23,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742097_1273 (size=14741) 2024-11-16T10:15:23,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/febad6b4d1e0468d85c1e74f1b78d49b 2024-11-16T10:15:23,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/14fb52410bff4213a6148de6bc2a8472 is 50, key is test_row_0/B:col10/1731752123651/Put/seqid=0 2024-11-16T10:15:23,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:23,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752183964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752183964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:23,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742098_1274 (size=12301) 2024-11-16T10:15:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:23,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/14fb52410bff4213a6148de6bc2a8472 2024-11-16T10:15:23,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752183974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:23,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752183975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:23,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752183980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:23,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/869018c167f34f07ab12037f6f6db647 is 50, key is test_row_0/C:col10/1731752123651/Put/seqid=0 2024-11-16T10:15:24,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742099_1275 (size=12301) 2024-11-16T10:15:24,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/869018c167f34f07ab12037f6f6db647 2024-11-16T10:15:24,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/febad6b4d1e0468d85c1e74f1b78d49b as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/febad6b4d1e0468d85c1e74f1b78d49b 2024-11-16T10:15:24,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/febad6b4d1e0468d85c1e74f1b78d49b, entries=200, sequenceid=324, filesize=14.4 K 2024-11-16T10:15:24,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/14fb52410bff4213a6148de6bc2a8472 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/14fb52410bff4213a6148de6bc2a8472 2024-11-16T10:15:24,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/14fb52410bff4213a6148de6bc2a8472, entries=150, sequenceid=324, filesize=12.0 K 2024-11-16T10:15:24,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/869018c167f34f07ab12037f6f6db647 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/869018c167f34f07ab12037f6f6db647 2024-11-16T10:15:24,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/869018c167f34f07ab12037f6f6db647, entries=150, sequenceid=324, filesize=12.0 K 2024-11-16T10:15:24,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=46.96 KB/48090 for dcee08b7bd0f481f30af9d9b29c726c5 in 117ms, sequenceid=324, compaction requested=true 2024-11-16T10:15:24,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:24,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:15:24,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:24,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:15:24,056 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:24,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:24,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:15:24,056 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:24,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:24,058 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:24,058 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/B is initiating minor compaction (all files) 2024-11-16T10:15:24,058 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/B in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:24,058 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/64c948eaee2a4a50b83a1c4af2efc6a3, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/23666c9856494979b4ede6dba94febae, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/14fb52410bff4213a6148de6bc2a8472] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=36.5 K 2024-11-16T10:15:24,058 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:24,058 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/A is initiating minor compaction (all files) 2024-11-16T10:15:24,058 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/A in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:24,059 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/ca9905a026fb4e18a97444f87d9a0870, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/5e8673ca927e4cb993309134e4958043, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/febad6b4d1e0468d85c1e74f1b78d49b] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=38.9 K 2024-11-16T10:15:24,060 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64c948eaee2a4a50b83a1c4af2efc6a3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731752121809 2024-11-16T10:15:24,060 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting ca9905a026fb4e18a97444f87d9a0870, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731752121809 2024-11-16T10:15:24,061 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23666c9856494979b4ede6dba94febae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731752122449 2024-11-16T10:15:24,061 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e8673ca927e4cb993309134e4958043, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731752122449 2024-11-16T10:15:24,061 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting febad6b4d1e0468d85c1e74f1b78d49b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731752123640 2024-11-16T10:15:24,061 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14fb52410bff4213a6148de6bc2a8472, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731752123640 2024-11-16T10:15:24,075 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#B#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:24,075 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#A#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:24,075 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/3c3b22bc8dae467287d9b513ab928250 is 50, key is test_row_0/B:col10/1731752123651/Put/seqid=0 2024-11-16T10:15:24,076 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/be98628ac57c4917a5eb5b25497ad993 is 50, key is test_row_0/A:col10/1731752123651/Put/seqid=0 2024-11-16T10:15:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:24,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T10:15:24,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:24,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:24,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:24,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:24,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:24,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:24,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/0d768904bd0a4093acddee017ddb33d2 is 50, key is test_row_0/A:col10/1731752123962/Put/seqid=0 2024-11-16T10:15:24,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742100_1276 (size=12881) 2024-11-16T10:15:24,117 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/be98628ac57c4917a5eb5b25497ad993 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/be98628ac57c4917a5eb5b25497ad993 2024-11-16T10:15:24,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742101_1277 (size=12881) 2024-11-16T10:15:24,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742102_1278 (size=14741) 2024-11-16T10:15:24,135 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/A of dcee08b7bd0f481f30af9d9b29c726c5 into be98628ac57c4917a5eb5b25497ad993(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:24,135 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:24,135 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/A, priority=13, startTime=1731752124056; duration=0sec 2024-11-16T10:15:24,135 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:24,135 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:A 2024-11-16T10:15:24,136 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:24,137 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:24,137 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/C is initiating minor compaction (all files) 2024-11-16T10:15:24,137 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/C in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:24,137 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/1e0d9e221d68437e8a2298c0aa743675, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/1c41b19cc02f403d9e748731ee1597df, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/869018c167f34f07ab12037f6f6db647] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=36.5 K 2024-11-16T10:15:24,137 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e0d9e221d68437e8a2298c0aa743675, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731752121809 2024-11-16T10:15:24,138 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c41b19cc02f403d9e748731ee1597df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731752122449 2024-11-16T10:15:24,138 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 869018c167f34f07ab12037f6f6db647, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731752123640 2024-11-16T10:15:24,147 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#C#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:24,148 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/e9a1a3a9f80d4e9ca31f52ecfc3c4840 is 50, key is test_row_0/C:col10/1731752123651/Put/seqid=0 2024-11-16T10:15:24,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742103_1279 (size=12881) 2024-11-16T10:15:24,162 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/e9a1a3a9f80d4e9ca31f52ecfc3c4840 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/e9a1a3a9f80d4e9ca31f52ecfc3c4840 2024-11-16T10:15:24,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752184159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:24,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752124142, max timestamp=1731752124164Num uniques 859; 2024-11-16T10:15:24,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752124142, max timestamp=1731752124164Num uniques 859; 2024-11-16T10:15:24,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:15:24,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:15:24,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752124142, max timestamp=1731752124164Num uniques 859; 2024-11-16T10:15:24,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:15:24,183 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/C of dcee08b7bd0f481f30af9d9b29c726c5 into e9a1a3a9f80d4e9ca31f52ecfc3c4840(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:24,183 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:24,183 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/C, priority=13, startTime=1731752124056; duration=0sec 2024-11-16T10:15:24,184 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:24,184 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:C 2024-11-16T10:15:24,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:24,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752184264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:24,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:24,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752184274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:24,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:24,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752184284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:24,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:24,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:24,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752184284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:24,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752184294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:24,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752184374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:24,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T10:15:24,394 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:15:24,394 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:15:24,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-16T10:15:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T10:15:24,396 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:15:24,396 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:15:24,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:15:24,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752184484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T10:15:24,530 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/3c3b22bc8dae467287d9b513ab928250 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/3c3b22bc8dae467287d9b513ab928250 2024-11-16T10:15:24,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/0d768904bd0a4093acddee017ddb33d2 2024-11-16T10:15:24,535 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/B of dcee08b7bd0f481f30af9d9b29c726c5 into 3c3b22bc8dae467287d9b513ab928250(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:24,535 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:24,535 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/B, priority=13, startTime=1731752124056; duration=0sec 2024-11-16T10:15:24,535 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:24,535 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:B 2024-11-16T10:15:24,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/c673c28a377a4bbc9fe508fb9ae58a13 is 50, key is test_row_0/B:col10/1731752123962/Put/seqid=0 2024-11-16T10:15:24,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742104_1280 (size=12301) 2024-11-16T10:15:24,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T10:15:24,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:24,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:24,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:24,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:24,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:24,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752184584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:24,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T10:15:24,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:24,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:24,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:24,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:24,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T10:15:24,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752184794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:24,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752184794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:24,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752184794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:24,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752184803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:24,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T10:15:24,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:24,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:24,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:24,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:24,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:24,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752184894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:24,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:24,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:24,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/c673c28a377a4bbc9fe508fb9ae58a13 2024-11-16T10:15:24,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/41f24033debb4578b53e01c3839f51ec is 50, key is test_row_0/C:col10/1731752123962/Put/seqid=0 2024-11-16T10:15:24,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742105_1281 (size=12301) 2024-11-16T10:15:25,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T10:15:25,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:25,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:25,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:25,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:25,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:25,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T10:15:25,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T10:15:25,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:25,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:25,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:25,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:25,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:25,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:25,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T10:15:25,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:25,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:25,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:25,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:25,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:25,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:25,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:25,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752185314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:25,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/41f24033debb4578b53e01c3839f51ec 2024-11-16T10:15:25,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/0d768904bd0a4093acddee017ddb33d2 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/0d768904bd0a4093acddee017ddb33d2 2024-11-16T10:15:25,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/0d768904bd0a4093acddee017ddb33d2, entries=200, sequenceid=335, filesize=14.4 K 2024-11-16T10:15:25,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/c673c28a377a4bbc9fe508fb9ae58a13 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/c673c28a377a4bbc9fe508fb9ae58a13 2024-11-16T10:15:25,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/c673c28a377a4bbc9fe508fb9ae58a13, entries=150, sequenceid=335, filesize=12.0 K 2024-11-16T10:15:25,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/41f24033debb4578b53e01c3839f51ec as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/41f24033debb4578b53e01c3839f51ec 2024-11-16T10:15:25,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/41f24033debb4578b53e01c3839f51ec, entries=150, sequenceid=335, filesize=12.0 K 2024-11-16T10:15:25,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for dcee08b7bd0f481f30af9d9b29c726c5 in 1292ms, sequenceid=335, compaction requested=false 2024-11-16T10:15:25,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:25,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:25,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-16T10:15:25,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:25,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:25,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:25,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:25,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:25,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:25,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/0b4e1e2281534eecb70a3ed476a68247 is 50, key is test_row_0/A:col10/1731752125405/Put/seqid=0 2024-11-16T10:15:25,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742106_1282 (size=14741) 2024-11-16T10:15:25,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752185448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:25,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T10:15:25,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:25,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:25,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:25,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:25,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:25,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T10:15:25,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:25,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752185563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:25,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T10:15:25,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:25,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:25,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:25,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:25,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:25,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T10:15:25,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:25,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:25,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:25,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:25,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:25,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:25,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752185777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:25,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/0b4e1e2281534eecb70a3ed476a68247 2024-11-16T10:15:25,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752185814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:25,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752185814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:25,819 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_2' in dcee08b7bd0f481f30af9d9b29c726c5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:15:25,819 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_0' in dcee08b7bd0f481f30af9d9b29c726c5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:15:25,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:25,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:25,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:25,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:25,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:25,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:25,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752185823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:25,829 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in dcee08b7bd0f481f30af9d9b29c726c5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:15:25,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:25,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:25,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:25,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/a75bd5cef3e04bafae70c91032cd90e5 is 50, key is test_row_0/B:col10/1731752125405/Put/seqid=0 2024-11-16T10:15:25,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742107_1283 (size=12301) 2024-11-16T10:15:25,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/a75bd5cef3e04bafae70c91032cd90e5 2024-11-16T10:15:25,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/88f700f1b48446379ad8cd3b55202bf6 is 50, key is test_row_0/C:col10/1731752125405/Put/seqid=0 2024-11-16T10:15:25,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742108_1284 (size=12301) 2024-11-16T10:15:25,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/88f700f1b48446379ad8cd3b55202bf6 2024-11-16T10:15:25,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/0b4e1e2281534eecb70a3ed476a68247 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/0b4e1e2281534eecb70a3ed476a68247 2024-11-16T10:15:25,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/0b4e1e2281534eecb70a3ed476a68247, entries=200, sequenceid=375, filesize=14.4 K 2024-11-16T10:15:25,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/a75bd5cef3e04bafae70c91032cd90e5 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/a75bd5cef3e04bafae70c91032cd90e5 2024-11-16T10:15:25,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/a75bd5cef3e04bafae70c91032cd90e5, entries=150, sequenceid=375, filesize=12.0 K 2024-11-16T10:15:25,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/88f700f1b48446379ad8cd3b55202bf6 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/88f700f1b48446379ad8cd3b55202bf6 2024-11-16T10:15:25,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/88f700f1b48446379ad8cd3b55202bf6, entries=150, sequenceid=375, filesize=12.0 K 2024-11-16T10:15:25,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for dcee08b7bd0f481f30af9d9b29c726c5 in 514ms, sequenceid=375, compaction requested=true 2024-11-16T10:15:25,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:25,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:15:25,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:25,920 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:25,920 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:25,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:15:25,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:25,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:15:25,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:25,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T10:15:25,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:25,924 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T10:15:25,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:25,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:25,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:25,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:25,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:25,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:25,925 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:25,925 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/B is initiating minor compaction (all files) 2024-11-16T10:15:25,925 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/B in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:25,925 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/3c3b22bc8dae467287d9b513ab928250, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/c673c28a377a4bbc9fe508fb9ae58a13, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/a75bd5cef3e04bafae70c91032cd90e5] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=36.6 K 2024-11-16T10:15:25,926 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:25,926 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/A is initiating minor compaction (all files) 2024-11-16T10:15:25,926 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/A in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:25,926 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/be98628ac57c4917a5eb5b25497ad993, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/0d768904bd0a4093acddee017ddb33d2, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/0b4e1e2281534eecb70a3ed476a68247] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=41.4 K 2024-11-16T10:15:25,926 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting be98628ac57c4917a5eb5b25497ad993, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731752123640 2024-11-16T10:15:25,927 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c3b22bc8dae467287d9b513ab928250, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731752123640 2024-11-16T10:15:25,927 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d768904bd0a4093acddee017ddb33d2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731752123948 2024-11-16T10:15:25,927 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting c673c28a377a4bbc9fe508fb9ae58a13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731752123948 2024-11-16T10:15:25,928 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b4e1e2281534eecb70a3ed476a68247, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731752124238 2024-11-16T10:15:25,929 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting a75bd5cef3e04bafae70c91032cd90e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731752124238 2024-11-16T10:15:25,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/49f47478a4b7453ebacfa8693d9f23a6 is 50, key is test_row_0/A:col10/1731752125446/Put/seqid=0 2024-11-16T10:15:25,951 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#A#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:25,952 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/efc46ce7ef074cd598848bf8b5e5ab0b is 50, key is test_row_0/A:col10/1731752125405/Put/seqid=0 2024-11-16T10:15:25,955 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#B#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:25,956 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/4a9c27db8a074be58dea07309154a66f is 50, key is test_row_0/B:col10/1731752125405/Put/seqid=0 2024-11-16T10:15:25,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742109_1285 (size=12301) 2024-11-16T10:15:25,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742110_1286 (size=12983) 2024-11-16T10:15:25,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742111_1287 (size=12983) 2024-11-16T10:15:26,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:26,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:26,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752126185, max timestamp=1731752126216Num uniques 859; 2024-11-16T10:15:26,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:15:26,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752126185, max timestamp=1731752126216Num uniques 859; 2024-11-16T10:15:26,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752126185, max timestamp=1731752126216Num uniques 859; 2024-11-16T10:15:26,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:15:26,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:15:26,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752186329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:26,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752186333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:26,370 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/49f47478a4b7453ebacfa8693d9f23a6 2024-11-16T10:15:26,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/015ef4c24bfc46ba9e8df419163b3f46 is 50, key is test_row_0/B:col10/1731752125446/Put/seqid=0 2024-11-16T10:15:26,387 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/4a9c27db8a074be58dea07309154a66f as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/4a9c27db8a074be58dea07309154a66f 2024-11-16T10:15:26,388 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/efc46ce7ef074cd598848bf8b5e5ab0b as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/efc46ce7ef074cd598848bf8b5e5ab0b 2024-11-16T10:15:26,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742112_1288 (size=12301) 2024-11-16T10:15:26,389 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/015ef4c24bfc46ba9e8df419163b3f46 2024-11-16T10:15:26,394 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/A of dcee08b7bd0f481f30af9d9b29c726c5 into efc46ce7ef074cd598848bf8b5e5ab0b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:26,394 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:26,394 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/A, priority=13, startTime=1731752125920; duration=0sec 2024-11-16T10:15:26,394 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:26,394 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:A 2024-11-16T10:15:26,394 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:26,395 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/B of dcee08b7bd0f481f30af9d9b29c726c5 into 4a9c27db8a074be58dea07309154a66f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:26,395 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:26,396 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/B, priority=13, startTime=1731752125920; duration=0sec 2024-11-16T10:15:26,396 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:26,396 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:B 2024-11-16T10:15:26,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/92785a21e15d432d89a0d3ac8e2b31c1 is 50, key is test_row_0/C:col10/1731752125446/Put/seqid=0 2024-11-16T10:15:26,400 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:26,400 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/C is initiating minor compaction (all files) 2024-11-16T10:15:26,400 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/C in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:26,401 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/e9a1a3a9f80d4e9ca31f52ecfc3c4840, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/41f24033debb4578b53e01c3839f51ec, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/88f700f1b48446379ad8cd3b55202bf6] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=36.6 K 2024-11-16T10:15:26,401 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9a1a3a9f80d4e9ca31f52ecfc3c4840, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731752123640 2024-11-16T10:15:26,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742113_1289 (size=12301) 2024-11-16T10:15:26,401 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41f24033debb4578b53e01c3839f51ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731752123948 2024-11-16T10:15:26,402 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88f700f1b48446379ad8cd3b55202bf6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731752124238 2024-11-16T10:15:26,403 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/92785a21e15d432d89a0d3ac8e2b31c1 2024-11-16T10:15:26,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/49f47478a4b7453ebacfa8693d9f23a6 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/49f47478a4b7453ebacfa8693d9f23a6 2024-11-16T10:15:26,419 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/49f47478a4b7453ebacfa8693d9f23a6, entries=150, sequenceid=385, filesize=12.0 K 2024-11-16T10:15:26,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/015ef4c24bfc46ba9e8df419163b3f46 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/015ef4c24bfc46ba9e8df419163b3f46 2024-11-16T10:15:26,423 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#C#compaction#248 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:26,424 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/12d3c799df7b4a3f90a9a1f21f20b77c is 50, key is test_row_0/C:col10/1731752125405/Put/seqid=0 2024-11-16T10:15:26,430 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/015ef4c24bfc46ba9e8df419163b3f46, entries=150, sequenceid=385, filesize=12.0 K 2024-11-16T10:15:26,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/92785a21e15d432d89a0d3ac8e2b31c1 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/92785a21e15d432d89a0d3ac8e2b31c1 2024-11-16T10:15:26,436 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/92785a21e15d432d89a0d3ac8e2b31c1, entries=150, sequenceid=385, filesize=12.0 K 2024-11-16T10:15:26,437 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for dcee08b7bd0f481f30af9d9b29c726c5 in 513ms, sequenceid=385, compaction requested=false 2024-11-16T10:15:26,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:26,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:26,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-16T10:15:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-16T10:15:26,442 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-16T10:15:26,442 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0440 sec 2024-11-16T10:15:26,451 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.0530 sec 2024-11-16T10:15:26,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731752126295, max timestamp=1731752126448Num uniques 1018; 2024-11-16T10:15:26,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-16T10:15:26,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731752126295, max timestamp=1731752126449Num uniques 1018; 2024-11-16T10:15:26,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-16T10:15:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:26,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731752126295, max timestamp=1731752126449Num uniques 1018; 2024-11-16T10:15:26,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=71.56 KB heapSize=198.72 KB 2024-11-16T10:15:26,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-16T10:15:26,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:26,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:26,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:26,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:26,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:26,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:26,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/a8b56b157eb541ffbdec3130091f7d67 is 50, key is test_row_0/A:col10/1731752126448/Put/seqid=0 2024-11-16T10:15:26,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742114_1290 (size=12983) 2024-11-16T10:15:26,469 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/12d3c799df7b4a3f90a9a1f21f20b77c as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/12d3c799df7b4a3f90a9a1f21f20b77c 2024-11-16T10:15:26,477 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/C of dcee08b7bd0f481f30af9d9b29c726c5 into 12d3c799df7b4a3f90a9a1f21f20b77c(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:26,478 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:26,478 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/C, priority=13, startTime=1731752125922; duration=0sec 2024-11-16T10:15:26,478 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:26,478 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:C 2024-11-16T10:15:26,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742115_1291 (size=17181) 2024-11-16T10:15:26,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752126507, max timestamp=1731752126532Num uniques 859; 2024-11-16T10:15:26,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752126507, max timestamp=1731752126532Num uniques 859; 2024-11-16T10:15:26,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752126507, max timestamp=1731752126532Num uniques 859; 2024-11-16T10:15:26,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:15:26,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:15:26,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:15:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T10:15:26,544 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:15:26,545 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:15:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-16T10:15:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T10:15:26,548 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:15:26,549 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:15:26,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:15:26,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731752126593, max timestamp=1731752126605Num uniques 1018; 2024-11-16T10:15:26,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7683, totalHeapSize=17712, min timestamp=1731752126593, max timestamp=1731752126606Num uniques 1018; 2024-11-16T10:15:26,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-16T10:15:26,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46613, new segment data size=7683, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-16T10:15:26,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731752126593, max timestamp=1731752126605Num uniques 1018; 2024-11-16T10:15:26,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-16T10:15:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T10:15:26,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731752126646, max timestamp=1731752126676Num uniques 1027; 2024-11-16T10:15:26,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731752126646, max timestamp=1731752126676Num uniques 1027; 2024-11-16T10:15:26,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731752126646, max timestamp=1731752126676Num uniques 1027; 2024-11-16T10:15:26,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-16T10:15:26,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-16T10:15:26,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-16T10:15:26,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T10:15:26,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:26,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:26,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:26,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:26,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:26,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731752126703, max timestamp=1731752126740Num uniques 1086; 2024-11-16T10:15:26,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731752126703, max timestamp=1731752126740Num uniques 1086; 2024-11-16T10:15:26,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-16T10:15:26,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-16T10:15:26,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731752126703, max timestamp=1731752126740Num uniques 1086; 2024-11-16T10:15:26,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-16T10:15:26,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731752126782, max timestamp=1731752126796Num uniques 1095; 2024-11-16T10:15:26,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731752126782, max timestamp=1731752126796Num uniques 1095; 2024-11-16T10:15:26,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-16T10:15:26,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-16T10:15:26,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731752126782, max timestamp=1731752126796Num uniques 1095; 2024-11-16T10:15:26,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-16T10:15:26,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T10:15:26,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:26,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:26,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:26,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:26,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:26,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731752126829, max timestamp=1731752126856Num uniques 1054; 2024-11-16T10:15:26,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731752126829, max timestamp=1731752126857Num uniques 1054; 2024-11-16T10:15:26,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-16T10:15:26,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-16T10:15:26,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731752126829, max timestamp=1731752126856Num uniques 1054; 2024-11-16T10:15:26,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-16T10:15:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T10:15:26,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/a8b56b157eb541ffbdec3130091f7d67 2024-11-16T10:15:26,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/959ef2b1bbdb4ec4a492af26f10be3c7 is 50, key is test_row_0/B:col10/1731752126448/Put/seqid=0 2024-11-16T10:15:26,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742116_1292 (size=12301) 2024-11-16T10:15:26,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/959ef2b1bbdb4ec4a492af26f10be3c7 2024-11-16T10:15:26,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/c2687817c73d4b7495ce3d55f107a2f4 is 50, key is test_row_0/C:col10/1731752126448/Put/seqid=0 2024-11-16T10:15:26,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731752126867, max timestamp=1731752126917Num uniques 1013; 2024-11-16T10:15:26,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-16T10:15:26,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731752126867, max timestamp=1731752126917Num uniques 1013; 2024-11-16T10:15:26,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-16T10:15:26,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731752126867, max timestamp=1731752126917Num uniques 1013; 2024-11-16T10:15:26,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-16T10:15:26,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742117_1293 (size=12301) 2024-11-16T10:15:26,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731752126940, max timestamp=1731752126972Num uniques 1072; 2024-11-16T10:15:26,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-16T10:15:26,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731752126940, max timestamp=1731752126972Num uniques 1072; 2024-11-16T10:15:26,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-16T10:15:26,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731752126940, max timestamp=1731752126972Num uniques 1072; 2024-11-16T10:15:26,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-16T10:15:27,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T10:15:27,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:27,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:27,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:27,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:27,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:27,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731752126990, max timestamp=1731752127024Num uniques 1031; 2024-11-16T10:15:27,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731752126990, max timestamp=1731752127024Num uniques 1031; 2024-11-16T10:15:27,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731752126990, max timestamp=1731752127024Num uniques 1031; 2024-11-16T10:15:27,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-16T10:15:27,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-16T10:15:27,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-16T10:15:27,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731752127070, max timestamp=1731752127083Num uniques 1090; 2024-11-16T10:15:27,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-16T10:15:27,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731752127070, max timestamp=1731752127083Num uniques 1040; 2024-11-16T10:15:27,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731752127070, max timestamp=1731752127083Num uniques 1040; 2024-11-16T10:15:27,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-16T10:15:27,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-16T10:15:27,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731752127123, max timestamp=1731752127132Num uniques 1099; 2024-11-16T10:15:27,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=9114, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-11-16T10:15:27,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731752127123, max timestamp=1731752127132Num uniques 1099; 2024-11-16T10:15:27,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731752127123, max timestamp=1731752127132Num uniques 1049; 2024-11-16T10:15:27,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=6870, suffix heap size=137064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=150 2024-11-16T10:15:27,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=6870, suffix heap size=142264,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=150 2024-11-16T10:15:27,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T10:15:27,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:27,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:27,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:27,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:27,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T10:15:27,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9518, totalHeapSize=21872, min timestamp=1731752127182, max timestamp=1731752127197Num uniques 1058; 2024-11-16T10:15:27,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48448, new segment data size=9518, suffix heap size=138000,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-11-16T10:15:27,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9518, totalHeapSize=21872, min timestamp=1731752127182, max timestamp=1731752127197Num uniques 1009; 2024-11-16T10:15:27,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46204, new segment data size=9518, suffix heap size=132904,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=208 2024-11-16T10:15:27,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9518, totalHeapSize=21872, min timestamp=1731752127182, max timestamp=1731752127197Num uniques 1009; 2024-11-16T10:15:27,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46204, new segment data size=9518, suffix heap size=132904,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=208 2024-11-16T10:15:27,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731752127217, max timestamp=1731752127249Num uniques 1067; 2024-11-16T10:15:27,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-16T10:15:27,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731752127217, max timestamp=1731752127249Num uniques 1067; 2024-11-16T10:15:27,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731752127217, max timestamp=1731752127249Num uniques 1067; 2024-11-16T10:15:27,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-16T10:15:27,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-16T10:15:27,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T10:15:27,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:27,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:27,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:27,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:27,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:27,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/c2687817c73d4b7495ce3d55f107a2f4 2024-11-16T10:15:27,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/a8b56b157eb541ffbdec3130091f7d67 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/a8b56b157eb541ffbdec3130091f7d67 2024-11-16T10:15:27,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/a8b56b157eb541ffbdec3130091f7d67, entries=250, sequenceid=426, filesize=16.8 K 2024-11-16T10:15:27,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/959ef2b1bbdb4ec4a492af26f10be3c7 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/959ef2b1bbdb4ec4a492af26f10be3c7 2024-11-16T10:15:27,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/959ef2b1bbdb4ec4a492af26f10be3c7, entries=150, sequenceid=426, filesize=12.0 K 2024-11-16T10:15:27,340 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8046, totalHeapSize=18544, min timestamp=1731752127290, max timestamp=1731752127320Num uniques 1076; 2024-11-16T10:15:27,340 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8051, totalHeapSize=18544, min timestamp=1731752127290, max timestamp=1731752127320Num uniques 1076; 2024-11-16T10:15:27,340 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49266, new segment data size=8046, suffix heap size=139872,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=176 2024-11-16T10:15:27,340 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49271, new segment data size=8051, suffix heap size=139872,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=176 2024-11-16T10:15:27,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8054, totalHeapSize=18544, min timestamp=1731752127290, max timestamp=1731752127320Num uniques 1076; 2024-11-16T10:15:27,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49274, new segment data size=8054, suffix heap size=139872,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=176 2024-11-16T10:15:27,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/c2687817c73d4b7495ce3d55f107a2f4 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/c2687817c73d4b7495ce3d55f107a2f4 2024-11-16T10:15:27,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/c2687817c73d4b7495ce3d55f107a2f4, entries=150, sequenceid=426, filesize=12.0 K 2024-11-16T10:15:27,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=33.54 KB/34350 for dcee08b7bd0f481f30af9d9b29c726c5 in 892ms, sequenceid=426, compaction requested=true 2024-11-16T10:15:27,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:27,346 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:27,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:15:27,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:27,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:15:27,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:27,346 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:27,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:15:27,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:27,347 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:27,347 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/A is initiating minor compaction (all files) 2024-11-16T10:15:27,347 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/A in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:27,347 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/efc46ce7ef074cd598848bf8b5e5ab0b, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/49f47478a4b7453ebacfa8693d9f23a6, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/a8b56b157eb541ffbdec3130091f7d67] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=41.5 K 2024-11-16T10:15:27,347 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:27,347 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting efc46ce7ef074cd598848bf8b5e5ab0b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731752124238 2024-11-16T10:15:27,347 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/B is initiating minor compaction (all files) 2024-11-16T10:15:27,348 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/B in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:27,348 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/4a9c27db8a074be58dea07309154a66f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/015ef4c24bfc46ba9e8df419163b3f46, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/959ef2b1bbdb4ec4a492af26f10be3c7] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=36.7 K 2024-11-16T10:15:27,348 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49f47478a4b7453ebacfa8693d9f23a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731752125421 2024-11-16T10:15:27,348 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a9c27db8a074be58dea07309154a66f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731752124238 2024-11-16T10:15:27,348 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8b56b157eb541ffbdec3130091f7d67, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731752126295 2024-11-16T10:15:27,348 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 015ef4c24bfc46ba9e8df419163b3f46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731752125421 2024-11-16T10:15:27,349 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 959ef2b1bbdb4ec4a492af26f10be3c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731752126331 2024-11-16T10:15:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:27,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=53.67 KB heapSize=144.33 KB 2024-11-16T10:15:27,355 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#B#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:27,355 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#A#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:27,355 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/177a25ffef5542b1a2d5e3a4762e17a8 is 50, key is test_row_0/B:col10/1731752126448/Put/seqid=0 2024-11-16T10:15:27,356 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/9df5e2a91bb94fb9a023ac7e52e7b5cc is 50, key is test_row_0/A:col10/1731752126448/Put/seqid=0 2024-11-16T10:15:27,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:27,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:27,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:27,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:27,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:27,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:27,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/325d66fb84ec4554af167022abc67cba is 50, key is test_row_0/A:col10/1731752127353/Put/seqid=0 2024-11-16T10:15:27,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742118_1294 (size=13085) 2024-11-16T10:15:27,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742120_1296 (size=14741) 2024-11-16T10:15:27,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742119_1295 (size=13085) 2024-11-16T10:15:27,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752127393, max timestamp=1731752127417Num uniques 859; 2024-11-16T10:15:27,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:15:27,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752127393, max timestamp=1731752127417Num uniques 859; 2024-11-16T10:15:27,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:15:27,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752127393, max timestamp=1731752127417Num uniques 859; 2024-11-16T10:15:27,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:15:27,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T10:15:27,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:27,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:27,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:27,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:27,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:27,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752187475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:27,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:27,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:27,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:27,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752187481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:27,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:27,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:27,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:27,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752187584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:27,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752187594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:27,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:27,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:27,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T10:15:27,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:27,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:27,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:27,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:27,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T10:15:27,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T10:15:27,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:27,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:27,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:27,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:27,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:27,782 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/177a25ffef5542b1a2d5e3a4762e17a8 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/177a25ffef5542b1a2d5e3a4762e17a8 2024-11-16T10:15:27,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=676 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/325d66fb84ec4554af167022abc67cba 2024-11-16T10:15:27,793 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/B of dcee08b7bd0f481f30af9d9b29c726c5 into 177a25ffef5542b1a2d5e3a4762e17a8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:27,793 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:27,793 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/B, priority=13, startTime=1731752127346; duration=0sec 2024-11-16T10:15:27,793 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:27,793 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:B 2024-11-16T10:15:27,793 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:27,795 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/9df5e2a91bb94fb9a023ac7e52e7b5cc as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/9df5e2a91bb94fb9a023ac7e52e7b5cc 2024-11-16T10:15:27,795 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:27,795 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/C is initiating minor compaction (all files) 2024-11-16T10:15:27,795 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/C in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:27,795 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/12d3c799df7b4a3f90a9a1f21f20b77c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/92785a21e15d432d89a0d3ac8e2b31c1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/c2687817c73d4b7495ce3d55f107a2f4] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=36.7 K 2024-11-16T10:15:27,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/f66ae675f53a4f91b0055411b4627877 is 50, key is test_row_0/B:col10/1731752127353/Put/seqid=0 2024-11-16T10:15:27,796 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 12d3c799df7b4a3f90a9a1f21f20b77c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731752124238 2024-11-16T10:15:27,797 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 92785a21e15d432d89a0d3ac8e2b31c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731752125421 2024-11-16T10:15:27,797 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting c2687817c73d4b7495ce3d55f107a2f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731752126331 2024-11-16T10:15:27,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:27,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752187794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:27,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:27,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:27,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:27,803 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/A of dcee08b7bd0f481f30af9d9b29c726c5 into 9df5e2a91bb94fb9a023ac7e52e7b5cc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:27,803 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:27,803 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/A, priority=13, startTime=1731752127346; duration=0sec 2024-11-16T10:15:27,803 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:27,803 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:A 2024-11-16T10:15:27,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752187804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:27,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:27,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:27,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:27,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742121_1297 (size=12301) 2024-11-16T10:15:27,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=676 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/f66ae675f53a4f91b0055411b4627877 2024-11-16T10:15:27,818 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#C#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:27,819 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/af8a2d0f514843c7b8147e82c7ba218c is 50, key is test_row_0/C:col10/1731752126448/Put/seqid=0 2024-11-16T10:15:27,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752187832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:27,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752187834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:27,837 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_0' in dcee08b7bd0f481f30af9d9b29c726c5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:15:27,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:27,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:27,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:27,838 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_2' in dcee08b7bd0f481f30af9d9b29c726c5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:15:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:27,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742122_1298 (size=13085) 2024-11-16T10:15:27,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/769fb5de49c34f7787cd90cef7debc3d is 50, key is test_row_0/C:col10/1731752127353/Put/seqid=0 2024-11-16T10:15:27,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:27,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742123_1299 (size=12301) 2024-11-16T10:15:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752187844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:27,850 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in dcee08b7bd0f481f30af9d9b29c726c5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:15:27,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:27,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:27,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:27,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T10:15:27,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:27,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:27,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:27,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:27,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:28,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T10:15:28,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:28,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:28,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:28,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:28,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:28,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752188104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:28,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752188114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:28,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:28,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:28,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:28,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T10:15:28,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:28,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:28,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:28,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:28,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:28,244 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/af8a2d0f514843c7b8147e82c7ba218c as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/af8a2d0f514843c7b8147e82c7ba218c 2024-11-16T10:15:28,248 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/C of dcee08b7bd0f481f30af9d9b29c726c5 into af8a2d0f514843c7b8147e82c7ba218c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:28,248 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:28,248 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/C, priority=13, startTime=1731752127346; duration=0sec 2024-11-16T10:15:28,248 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:28,248 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:C 2024-11-16T10:15:28,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=676 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/769fb5de49c34f7787cd90cef7debc3d 2024-11-16T10:15:28,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/325d66fb84ec4554af167022abc67cba as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/325d66fb84ec4554af167022abc67cba 2024-11-16T10:15:28,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/325d66fb84ec4554af167022abc67cba, entries=200, sequenceid=676, filesize=14.4 K 2024-11-16T10:15:28,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/f66ae675f53a4f91b0055411b4627877 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/f66ae675f53a4f91b0055411b4627877 2024-11-16T10:15:28,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/f66ae675f53a4f91b0055411b4627877, entries=150, sequenceid=676, filesize=12.0 K 2024-11-16T10:15:28,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/769fb5de49c34f7787cd90cef7debc3d as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/769fb5de49c34f7787cd90cef7debc3d 2024-11-16T10:15:28,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/769fb5de49c34f7787cd90cef7debc3d, entries=150, sequenceid=676, filesize=12.0 K 2024-11-16T10:15:28,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~164.20 KB/168144, currentSize=123.00 KB/125950 for dcee08b7bd0f481f30af9d9b29c726c5 in 959ms, sequenceid=676, compaction requested=false 2024-11-16T10:15:28,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:28,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T10:15:28,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:28,387 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=123.00 KB heapSize=350.22 KB 2024-11-16T10:15:28,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:28,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:28,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:28,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:28,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:28,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:28,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/9261f56acd9a4af09abef4ad9e9a0250 is 50, key is test_row_0/A:col10/1731752127474/Put/seqid=0 2024-11-16T10:15:28,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742124_1300 (size=12301) 2024-11-16T10:15:28,396 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/9261f56acd9a4af09abef4ad9e9a0250 2024-11-16T10:15:28,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/fd373d7553f3434d9baffc4c7c488622 is 50, key is test_row_0/B:col10/1731752127474/Put/seqid=0 2024-11-16T10:15:28,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742125_1301 (size=12301) 2024-11-16T10:15:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:28,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:28,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752188638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:28,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:28,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:28,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:28,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752188641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:28,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:28,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:28,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T10:15:28,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752188744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:28,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:28,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:28,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:28,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752188753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:28,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:28,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:28,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:28,828 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/fd373d7553f3434d9baffc4c7c488622 2024-11-16T10:15:28,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/97dbb3d0559246b79c919f1fbc382d43 is 50, key is test_row_0/C:col10/1731752127474/Put/seqid=0 2024-11-16T10:15:28,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742126_1302 (size=12301) 2024-11-16T10:15:28,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:28,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752188954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:28,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:28,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752188965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:29,242 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/97dbb3d0559246b79c919f1fbc382d43 2024-11-16T10:15:29,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/9261f56acd9a4af09abef4ad9e9a0250 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/9261f56acd9a4af09abef4ad9e9a0250 2024-11-16T10:15:29,253 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/9261f56acd9a4af09abef4ad9e9a0250, entries=150, sequenceid=714, filesize=12.0 K 2024-11-16T10:15:29,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/fd373d7553f3434d9baffc4c7c488622 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/fd373d7553f3434d9baffc4c7c488622 2024-11-16T10:15:29,257 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/fd373d7553f3434d9baffc4c7c488622, entries=150, sequenceid=714, filesize=12.0 K 2024-11-16T10:15:29,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/97dbb3d0559246b79c919f1fbc382d43 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/97dbb3d0559246b79c919f1fbc382d43 2024-11-16T10:15:29,263 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/97dbb3d0559246b79c919f1fbc382d43, entries=150, sequenceid=714, filesize=12.0 K 2024-11-16T10:15:29,264 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~123.00 KB/125950, heapSize ~350.17 KB/358576, currentSize=60.38 KB/61830 for dcee08b7bd0f481f30af9d9b29c726c5 in 877ms, sequenceid=714, compaction requested=true 2024-11-16T10:15:29,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:29,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:29,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-16T10:15:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-16T10:15:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:29,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T10:15:29,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:29,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:29,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:29,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:29,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:29,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:29,268 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-16T10:15:29,268 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7170 sec 2024-11-16T10:15:29,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.7240 sec 2024-11-16T10:15:29,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/a4f2489b72134d798f624ddcfaadd85e is 50, key is test_row_0/A:col10/1731752128634/Put/seqid=0 2024-11-16T10:15:29,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742127_1303 (size=14741) 2024-11-16T10:15:29,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=727 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/a4f2489b72134d798f624ddcfaadd85e 2024-11-16T10:15:29,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/6e86be9fa154487aa34fd7f854fd93af is 50, key is test_row_0/B:col10/1731752128634/Put/seqid=0 2024-11-16T10:15:29,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742128_1304 (size=12301) 2024-11-16T10:15:29,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752189317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:29,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:29,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:29,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:29,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 304 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752189323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:29,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752189434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:29,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:29,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:29,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:29,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 305 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752189434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:29,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:29,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:29,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:29,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752189643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:29,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 306 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752189644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:29,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:29,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:29,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:29,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=727 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/6e86be9fa154487aa34fd7f854fd93af 2024-11-16T10:15:29,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/79762ded297e4da3959b68a494cca8d3 is 50, key is test_row_0/C:col10/1731752128634/Put/seqid=0 2024-11-16T10:15:29,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742129_1305 (size=12301) 2024-11-16T10:15:29,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:29,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752189953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:29,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:29,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:29,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:29,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 307 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752189953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:29,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:30,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=727 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/79762ded297e4da3959b68a494cca8d3 2024-11-16T10:15:30,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/a4f2489b72134d798f624ddcfaadd85e as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/a4f2489b72134d798f624ddcfaadd85e 2024-11-16T10:15:30,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/a4f2489b72134d798f624ddcfaadd85e, entries=200, sequenceid=727, filesize=14.4 K 2024-11-16T10:15:30,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/6e86be9fa154487aa34fd7f854fd93af as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/6e86be9fa154487aa34fd7f854fd93af 2024-11-16T10:15:30,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/6e86be9fa154487aa34fd7f854fd93af, entries=150, sequenceid=727, filesize=12.0 K 2024-11-16T10:15:30,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/79762ded297e4da3959b68a494cca8d3 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/79762ded297e4da3959b68a494cca8d3 2024-11-16T10:15:30,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/79762ded297e4da3959b68a494cca8d3, entries=150, sequenceid=727, filesize=12.0 K 2024-11-16T10:15:30,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for dcee08b7bd0f481f30af9d9b29c726c5 in 858ms, sequenceid=727, compaction requested=true 2024-11-16T10:15:30,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:30,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:15:30,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:30,126 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:15:30,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:15:30,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:30,126 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:15:30,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:15:30,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:30,128 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:15:30,128 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54868 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:15:30,128 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/B is initiating minor compaction (all files) 2024-11-16T10:15:30,128 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/A is initiating minor compaction (all files) 2024-11-16T10:15:30,128 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/A in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:30,128 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/B in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:30,128 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/9df5e2a91bb94fb9a023ac7e52e7b5cc, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/325d66fb84ec4554af167022abc67cba, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/9261f56acd9a4af09abef4ad9e9a0250, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/a4f2489b72134d798f624ddcfaadd85e] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=53.6 K 2024-11-16T10:15:30,128 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/177a25ffef5542b1a2d5e3a4762e17a8, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/f66ae675f53a4f91b0055411b4627877, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/fd373d7553f3434d9baffc4c7c488622, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/6e86be9fa154487aa34fd7f854fd93af] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=48.8 K 2024-11-16T10:15:30,129 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 177a25ffef5542b1a2d5e3a4762e17a8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731752126331 2024-11-16T10:15:30,129 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9df5e2a91bb94fb9a023ac7e52e7b5cc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731752126331 2024-11-16T10:15:30,129 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 325d66fb84ec4554af167022abc67cba, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=676, earliestPutTs=1731752127307 2024-11-16T10:15:30,129 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting f66ae675f53a4f91b0055411b4627877, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=676, earliestPutTs=1731752127320 2024-11-16T10:15:30,129 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9261f56acd9a4af09abef4ad9e9a0250, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1731752127467 2024-11-16T10:15:30,129 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting fd373d7553f3434d9baffc4c7c488622, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1731752127467 2024-11-16T10:15:30,129 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4f2489b72134d798f624ddcfaadd85e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1731752128629 2024-11-16T10:15:30,129 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e86be9fa154487aa34fd7f854fd93af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1731752128634 2024-11-16T10:15:30,137 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#A#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:30,137 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/a3574d5aee4540ea94e76e9adb9909f2 is 50, key is test_row_0/A:col10/1731752128634/Put/seqid=0 2024-11-16T10:15:30,142 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#B#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:30,143 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/aae16e284fa54aa5a1c2dd224db0e347 is 50, key is test_row_0/B:col10/1731752128634/Put/seqid=0 2024-11-16T10:15:30,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742130_1306 (size=13221) 2024-11-16T10:15:30,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742131_1307 (size=13221) 2024-11-16T10:15:30,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:30,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T10:15:30,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:30,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:30,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:30,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:30,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:30,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:30,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/231ea934053a40088b9bce038164fc78 is 50, key is test_row_0/A:col10/1731752129316/Put/seqid=0 2024-11-16T10:15:30,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:30,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752190500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:30,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:30,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:30,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:30,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 313 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752190501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:30,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742132_1308 (size=14741) 2024-11-16T10:15:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:30,559 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/a3574d5aee4540ea94e76e9adb9909f2 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/a3574d5aee4540ea94e76e9adb9909f2 2024-11-16T10:15:30,564 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/aae16e284fa54aa5a1c2dd224db0e347 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/aae16e284fa54aa5a1c2dd224db0e347 2024-11-16T10:15:30,567 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/A of dcee08b7bd0f481f30af9d9b29c726c5 into a3574d5aee4540ea94e76e9adb9909f2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:30,568 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:30,568 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/A, priority=12, startTime=1731752130125; duration=0sec 2024-11-16T10:15:30,568 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:30,568 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:A 2024-11-16T10:15:30,568 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:15:30,570 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:15:30,570 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/C is initiating minor compaction (all files) 2024-11-16T10:15:30,570 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/C in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:30,570 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/af8a2d0f514843c7b8147e82c7ba218c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/769fb5de49c34f7787cd90cef7debc3d, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/97dbb3d0559246b79c919f1fbc382d43, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/79762ded297e4da3959b68a494cca8d3] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=48.8 K 2024-11-16T10:15:30,570 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting af8a2d0f514843c7b8147e82c7ba218c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731752126331 2024-11-16T10:15:30,571 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 769fb5de49c34f7787cd90cef7debc3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=676, earliestPutTs=1731752127320 2024-11-16T10:15:30,572 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97dbb3d0559246b79c919f1fbc382d43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1731752127467 2024-11-16T10:15:30,572 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/B of dcee08b7bd0f481f30af9d9b29c726c5 into aae16e284fa54aa5a1c2dd224db0e347(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:30,572 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79762ded297e4da3959b68a494cca8d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1731752128634 2024-11-16T10:15:30,572 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:30,572 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/B, priority=12, startTime=1731752130125; duration=0sec 2024-11-16T10:15:30,572 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:30,572 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:B 2024-11-16T10:15:30,583 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#C#compaction#267 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:30,583 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/0b92dacba94b4b879806a152d94f95ef is 50, key is test_row_0/C:col10/1731752128634/Put/seqid=0 2024-11-16T10:15:30,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742133_1309 (size=13221) 2024-11-16T10:15:30,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752190614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:30,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 314 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752190615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T10:15:30,714 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:15:30,715 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:15:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-16T10:15:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T10:15:30,717 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:15:30,719 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:15:30,719 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:15:30,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T10:15:30,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:30,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752190824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:30,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:30,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:30,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 315 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752190824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:30,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:30,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:30,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:30,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:30,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:30,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T10:15:30,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:30,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:30,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:30,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:30,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:30,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:30,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=747 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/231ea934053a40088b9bce038164fc78 2024-11-16T10:15:30,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/c95a1248d25d41c39dbdaf6f1fcb8cc4 is 50, key is test_row_0/B:col10/1731752129316/Put/seqid=0 2024-11-16T10:15:30,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742134_1310 (size=12301) 2024-11-16T10:15:30,991 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/0b92dacba94b4b879806a152d94f95ef as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/0b92dacba94b4b879806a152d94f95ef 2024-11-16T10:15:30,995 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/C of dcee08b7bd0f481f30af9d9b29c726c5 into 0b92dacba94b4b879806a152d94f95ef(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:30,995 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:30,995 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/C, priority=12, startTime=1731752130126; duration=0sec 2024-11-16T10:15:30,995 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:30,995 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:C 2024-11-16T10:15:31,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T10:15:31,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:31,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:31,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:31,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:31,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T10:15:31,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752191134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:31,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:31,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:31,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:31,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 316 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752191134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:31,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:31,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:31,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:31,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T10:15:31,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:31,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:31,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:31,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:31,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:31,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=747 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/c95a1248d25d41c39dbdaf6f1fcb8cc4 2024-11-16T10:15:31,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/166b5696520448e4bd373c59d05f33da is 50, key is test_row_0/C:col10/1731752129316/Put/seqid=0 2024-11-16T10:15:31,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742135_1311 (size=12301) 2024-11-16T10:15:31,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=747 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/166b5696520448e4bd373c59d05f33da 2024-11-16T10:15:31,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T10:15:31,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:31,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:31,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:31,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:31,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:31,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/231ea934053a40088b9bce038164fc78 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/231ea934053a40088b9bce038164fc78 2024-11-16T10:15:31,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/231ea934053a40088b9bce038164fc78, entries=200, sequenceid=747, filesize=14.4 K 2024-11-16T10:15:31,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/c95a1248d25d41c39dbdaf6f1fcb8cc4 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/c95a1248d25d41c39dbdaf6f1fcb8cc4 2024-11-16T10:15:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T10:15:31,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/c95a1248d25d41c39dbdaf6f1fcb8cc4, entries=150, sequenceid=747, filesize=12.0 K 2024-11-16T10:15:31,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/166b5696520448e4bd373c59d05f33da as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/166b5696520448e4bd373c59d05f33da 2024-11-16T10:15:31,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/166b5696520448e4bd373c59d05f33da, entries=150, sequenceid=747, filesize=12.0 K 2024-11-16T10:15:31,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for dcee08b7bd0f481f30af9d9b29c726c5 in 884ms, sequenceid=747, compaction requested=false 2024-11-16T10:15:31,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:31,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T10:15:31,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:31,489 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T10:15:31,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:31,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:31,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:31,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:31,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:31,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:31,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/3dcfaedeb98044afa8232a9294e7cd65 is 50, key is test_row_0/A:col10/1731752130499/Put/seqid=0 2024-11-16T10:15:31,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742136_1312 (size=12301) 2024-11-16T10:15:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:31,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:31,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:31,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 325 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752191689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:31,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:31,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:31,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:31,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752191692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:31,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:31,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 326 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752191804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:31,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:31,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:31,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:31,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752191804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:31,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:31,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:31,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:31,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:31,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:31,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54338 deadline: 1731752191844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:31,848 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_2' in dcee08b7bd0f481f30af9d9b29c726c5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:15:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:31,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T10:15:31,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:31,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54414 deadline: 1731752191854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:31,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:31,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731752191854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:31,856 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_0' in dcee08b7bd0f481f30af9d9b29c726c5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:15:31,856 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in dcee08b7bd0f481f30af9d9b29c726c5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:15:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:31,903 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=762 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/3dcfaedeb98044afa8232a9294e7cd65 2024-11-16T10:15:31,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/c41ea5bb98004eba85a8c76d753a7d66 is 50, key is test_row_0/B:col10/1731752130499/Put/seqid=0 2024-11-16T10:15:31,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742137_1313 (size=12301) 2024-11-16T10:15:32,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:32,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 327 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752192014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:32,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:32,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:32,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752192014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:32,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:32,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:32,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:32,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:32,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:32,314 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=762 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/c41ea5bb98004eba85a8c76d753a7d66 2024-11-16T10:15:32,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/46e3dd01c1e1472ca8eb266664edda19 is 50, key is test_row_0/C:col10/1731752130499/Put/seqid=0 2024-11-16T10:15:32,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742138_1314 (size=12301) 2024-11-16T10:15:32,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:32,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 328 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752192324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:32,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:32,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:32,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:32,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:32,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752192324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:32,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:32,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:32,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:32,724 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=762 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/46e3dd01c1e1472ca8eb266664edda19 2024-11-16T10:15:32,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/3dcfaedeb98044afa8232a9294e7cd65 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/3dcfaedeb98044afa8232a9294e7cd65 2024-11-16T10:15:32,731 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/3dcfaedeb98044afa8232a9294e7cd65, entries=150, sequenceid=762, filesize=12.0 K 2024-11-16T10:15:32,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/c41ea5bb98004eba85a8c76d753a7d66 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/c41ea5bb98004eba85a8c76d753a7d66 2024-11-16T10:15:32,735 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/c41ea5bb98004eba85a8c76d753a7d66, entries=150, sequenceid=762, filesize=12.0 K 2024-11-16T10:15:32,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/46e3dd01c1e1472ca8eb266664edda19 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/46e3dd01c1e1472ca8eb266664edda19 2024-11-16T10:15:32,739 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/46e3dd01c1e1472ca8eb266664edda19, entries=150, sequenceid=762, filesize=12.0 K 2024-11-16T10:15:32,740 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for dcee08b7bd0f481f30af9d9b29c726c5 in 1251ms, sequenceid=762, compaction requested=true 2024-11-16T10:15:32,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:32,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:32,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-16T10:15:32,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-16T10:15:32,743 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-16T10:15:32,743 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0220 sec 2024-11-16T10:15:32,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.0280 sec 2024-11-16T10:15:32,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:32,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-16T10:15:32,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:32,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:32,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:32,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:32,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:32,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:32,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/4c7f6644236144a3b2007036d886b83b is 50, key is test_row_0/A:col10/1731752131682/Put/seqid=0 2024-11-16T10:15:32,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742139_1315 (size=14741) 2024-11-16T10:15:32,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=783 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/4c7f6644236144a3b2007036d886b83b 2024-11-16T10:15:32,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/575745ce818e4383a2ed7fa2ecab9c34 is 50, key is test_row_0/B:col10/1731752131682/Put/seqid=0 2024-11-16T10:15:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T10:15:32,876 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:15:32,877 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:15:32,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-16T10:15:32,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T10:15:32,880 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:15:32,881 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:15:32,881 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:15:32,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:32,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 334 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752192878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:32,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:32,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:32,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:32,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752192879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:32,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:32,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:32,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:32,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:32,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742140_1316 (size=12301) 2024-11-16T10:15:32,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=783 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/575745ce818e4383a2ed7fa2ecab9c34 2024-11-16T10:15:32,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/f80bda994c574de8924cc91e667f1183 is 50, key is test_row_0/C:col10/1731752131682/Put/seqid=0 2024-11-16T10:15:32,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742141_1317 (size=12301) 2024-11-16T10:15:32,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=783 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/f80bda994c574de8924cc91e667f1183 2024-11-16T10:15:32,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/4c7f6644236144a3b2007036d886b83b as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/4c7f6644236144a3b2007036d886b83b 2024-11-16T10:15:32,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/4c7f6644236144a3b2007036d886b83b, entries=200, sequenceid=783, filesize=14.4 K 2024-11-16T10:15:32,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/575745ce818e4383a2ed7fa2ecab9c34 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/575745ce818e4383a2ed7fa2ecab9c34 2024-11-16T10:15:32,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/575745ce818e4383a2ed7fa2ecab9c34, entries=150, sequenceid=783, filesize=12.0 K 2024-11-16T10:15:32,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/f80bda994c574de8924cc91e667f1183 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/f80bda994c574de8924cc91e667f1183 2024-11-16T10:15:32,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/f80bda994c574de8924cc91e667f1183, entries=150, sequenceid=783, filesize=12.0 K 2024-11-16T10:15:32,972 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for dcee08b7bd0f481f30af9d9b29c726c5 in 135ms, sequenceid=783, compaction requested=true 2024-11-16T10:15:32,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:32,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:15:32,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:32,972 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:15:32,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:15:32,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:32,972 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:15:32,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:15:32,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:32,973 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:15:32,973 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:15:32,973 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/B is initiating minor compaction (all files) 2024-11-16T10:15:32,973 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/A is initiating minor compaction (all files) 2024-11-16T10:15:32,974 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/B in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:32,974 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/A in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:32,974 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/aae16e284fa54aa5a1c2dd224db0e347, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/c95a1248d25d41c39dbdaf6f1fcb8cc4, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/c41ea5bb98004eba85a8c76d753a7d66, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/575745ce818e4383a2ed7fa2ecab9c34] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=48.9 K 2024-11-16T10:15:32,974 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/a3574d5aee4540ea94e76e9adb9909f2, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/231ea934053a40088b9bce038164fc78, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/3dcfaedeb98044afa8232a9294e7cd65, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/4c7f6644236144a3b2007036d886b83b] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=53.7 K 2024-11-16T10:15:32,974 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting aae16e284fa54aa5a1c2dd224db0e347, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1731752128634 2024-11-16T10:15:32,974 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3574d5aee4540ea94e76e9adb9909f2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1731752128634 2024-11-16T10:15:32,974 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 231ea934053a40088b9bce038164fc78, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=747, earliestPutTs=1731752129315 2024-11-16T10:15:32,974 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting c95a1248d25d41c39dbdaf6f1fcb8cc4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=747, earliestPutTs=1731752129315 2024-11-16T10:15:32,976 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting c41ea5bb98004eba85a8c76d753a7d66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=762, earliestPutTs=1731752130484 2024-11-16T10:15:32,976 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dcfaedeb98044afa8232a9294e7cd65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=762, earliestPutTs=1731752130484 2024-11-16T10:15:32,976 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c7f6644236144a3b2007036d886b83b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=783, earliestPutTs=1731752131682 2024-11-16T10:15:32,976 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 575745ce818e4383a2ed7fa2ecab9c34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=783, earliestPutTs=1731752131682 2024-11-16T10:15:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T10:15:32,991 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#B#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:32,991 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/531362762a0f441fa76fefb83ef9464a is 50, key is test_row_0/B:col10/1731752131682/Put/seqid=0 2024-11-16T10:15:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:32,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T10:15:32,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:32,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:32,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:32,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:32,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:32,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:33,007 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#A#compaction#277 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:33,008 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/6ef493cf24b04424a899b7e53b92de91 is 50, key is test_row_0/A:col10/1731752131682/Put/seqid=0 2024-11-16T10:15:33,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-16T10:15:33,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:33,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/93e1d9a3e6ea4521acb8aeec94a0c870 is 50, key is test_row_0/A:col10/1731752132996/Put/seqid=0 2024-11-16T10:15:33,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:33,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:33,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:33,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:33,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:33,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742142_1318 (size=13357) 2024-11-16T10:15:33,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752193058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:33,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:33,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:33,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:33,073 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/531362762a0f441fa76fefb83ef9464a as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/531362762a0f441fa76fefb83ef9464a 2024-11-16T10:15:33,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 344 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752193068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:33,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:33,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:33,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:33,082 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/B of dcee08b7bd0f481f30af9d9b29c726c5 into 531362762a0f441fa76fefb83ef9464a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:33,082 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:33,082 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/B, priority=12, startTime=1731752132972; duration=0sec 2024-11-16T10:15:33,082 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:33,082 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:B 2024-11-16T10:15:33,082 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:15:33,084 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:15:33,084 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/C is initiating minor compaction (all files) 2024-11-16T10:15:33,084 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/C in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:33,084 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/0b92dacba94b4b879806a152d94f95ef, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/166b5696520448e4bd373c59d05f33da, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/46e3dd01c1e1472ca8eb266664edda19, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/f80bda994c574de8924cc91e667f1183] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=48.9 K 2024-11-16T10:15:33,085 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b92dacba94b4b879806a152d94f95ef, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1731752128634 2024-11-16T10:15:33,086 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 166b5696520448e4bd373c59d05f33da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=747, earliestPutTs=1731752129315 2024-11-16T10:15:33,086 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 46e3dd01c1e1472ca8eb266664edda19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=762, earliestPutTs=1731752130484 2024-11-16T10:15:33,086 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting f80bda994c574de8924cc91e667f1183, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=783, earliestPutTs=1731752131682 2024-11-16T10:15:33,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742143_1319 (size=13357) 2024-11-16T10:15:33,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742144_1320 (size=12301) 2024-11-16T10:15:33,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=796 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/93e1d9a3e6ea4521acb8aeec94a0c870 2024-11-16T10:15:33,110 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#C#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:33,111 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/6ca6f708835c4956a42e1c5215be1188 is 50, key is test_row_0/C:col10/1731752131682/Put/seqid=0 2024-11-16T10:15:33,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/eab06b7ca5944051a29d33536be52a8a is 50, key is test_row_0/B:col10/1731752132996/Put/seqid=0 2024-11-16T10:15:33,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742145_1321 (size=13357) 2024-11-16T10:15:33,169 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/6ca6f708835c4956a42e1c5215be1188 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/6ca6f708835c4956a42e1c5215be1188 2024-11-16T10:15:33,175 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/C of dcee08b7bd0f481f30af9d9b29c726c5 into 6ca6f708835c4956a42e1c5215be1188(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:33,175 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:33,175 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/C, priority=12, startTime=1731752132972; duration=0sec 2024-11-16T10:15:33,175 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:33,175 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:C 2024-11-16T10:15:33,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:33,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752193174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:33,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:33,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 345 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752193184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:33,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:33,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:33,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:33,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-16T10:15:33,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:33,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:33,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:33,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:33,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:33,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:33,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T10:15:33,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742146_1322 (size=12301) 2024-11-16T10:15:33,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=796 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/eab06b7ca5944051a29d33536be52a8a 2024-11-16T10:15:33,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/6bc7b64537c342c196ec67d7e4e463f1 is 50, key is test_row_0/C:col10/1731752132996/Put/seqid=0 2024-11-16T10:15:33,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742147_1323 (size=12301) 2024-11-16T10:15:33,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-16T10:15:33,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:33,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:33,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:33,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:33,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:33,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:33,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:33,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752193384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:33,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:33,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:33,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:33,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:33,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 346 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752193394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:33,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:33,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:33,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:33,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-16T10:15:33,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:33,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:33,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:33,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:33,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:33,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:33,501 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/6ef493cf24b04424a899b7e53b92de91 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/6ef493cf24b04424a899b7e53b92de91 2024-11-16T10:15:33,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T10:15:33,507 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/A of dcee08b7bd0f481f30af9d9b29c726c5 into 6ef493cf24b04424a899b7e53b92de91(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:33,507 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:33,507 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/A, priority=12, startTime=1731752132972; duration=0sec 2024-11-16T10:15:33,507 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:33,508 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:A 2024-11-16T10:15:33,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=796 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/6bc7b64537c342c196ec67d7e4e463f1 2024-11-16T10:15:33,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/93e1d9a3e6ea4521acb8aeec94a0c870 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/93e1d9a3e6ea4521acb8aeec94a0c870 2024-11-16T10:15:33,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/93e1d9a3e6ea4521acb8aeec94a0c870, entries=150, sequenceid=796, filesize=12.0 K 2024-11-16T10:15:33,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/eab06b7ca5944051a29d33536be52a8a as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/eab06b7ca5944051a29d33536be52a8a 2024-11-16T10:15:33,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/eab06b7ca5944051a29d33536be52a8a, entries=150, sequenceid=796, filesize=12.0 K 2024-11-16T10:15:33,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/6bc7b64537c342c196ec67d7e4e463f1 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/6bc7b64537c342c196ec67d7e4e463f1 2024-11-16T10:15:33,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/6bc7b64537c342c196ec67d7e4e463f1, entries=150, sequenceid=796, filesize=12.0 K 2024-11-16T10:15:33,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for dcee08b7bd0f481f30af9d9b29c726c5 in 643ms, sequenceid=796, compaction requested=false 2024-11-16T10:15:33,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:33,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-16T10:15:33,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:33,646 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T10:15:33,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:33,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:33,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:33,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:33,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:33,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:33,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/bc86c67cbaec4d04b17d1ffb1e2fa0f3 is 50, key is test_row_0/A:col10/1731752133055/Put/seqid=0 2024-11-16T10:15:33,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742148_1324 (size=12301) 2024-11-16T10:15:33,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:33,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. as already flushing 2024-11-16T10:15:33,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752193719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:33,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:33,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:33,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:33,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:33,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 351 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752193722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:33,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:33,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:33,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:33,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752193834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:33,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 352 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752193834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:34,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T10:15:34,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 302 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752194044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:34,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 353 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752194044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:34,054 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=818 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/bc86c67cbaec4d04b17d1ffb1e2fa0f3 2024-11-16T10:15:34,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/2f2eb6ae3d844ff59c78b7a0d02b8577 is 50, key is test_row_0/B:col10/1731752133055/Put/seqid=0 2024-11-16T10:15:34,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742149_1325 (size=12301) 2024-11-16T10:15:34,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:34,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 303 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1731752194354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:34,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:34,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:34,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:34,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 354 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54400 deadline: 1731752194354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:34,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:34,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:34,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:34,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:15:34,467 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=818 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/2f2eb6ae3d844ff59c78b7a0d02b8577 2024-11-16T10:15:34,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/f5d7b66f6d0e4344bcb1fc4a01b8c283 is 50, key is test_row_0/C:col10/1731752133055/Put/seqid=0 2024-11-16T10:15:34,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742150_1326 (size=12301) 2024-11-16T10:15:34,498 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=818 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/f5d7b66f6d0e4344bcb1fc4a01b8c283 2024-11-16T10:15:34,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/bc86c67cbaec4d04b17d1ffb1e2fa0f3 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/bc86c67cbaec4d04b17d1ffb1e2fa0f3 2024-11-16T10:15:34,509 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/bc86c67cbaec4d04b17d1ffb1e2fa0f3, entries=150, sequenceid=818, filesize=12.0 K 2024-11-16T10:15:34,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/2f2eb6ae3d844ff59c78b7a0d02b8577 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/2f2eb6ae3d844ff59c78b7a0d02b8577 2024-11-16T10:15:34,516 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/2f2eb6ae3d844ff59c78b7a0d02b8577, entries=150, sequenceid=818, filesize=12.0 K 2024-11-16T10:15:34,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/f5d7b66f6d0e4344bcb1fc4a01b8c283 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/f5d7b66f6d0e4344bcb1fc4a01b8c283 2024-11-16T10:15:34,523 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/f5d7b66f6d0e4344bcb1fc4a01b8c283, entries=150, sequenceid=818, filesize=12.0 K 2024-11-16T10:15:34,524 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for dcee08b7bd0f481f30af9d9b29c726c5 in 878ms, sequenceid=818, compaction requested=true 2024-11-16T10:15:34,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:34,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:34,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-16T10:15:34,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-16T10:15:34,527 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-16T10:15:34,527 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6450 sec 2024-11-16T10:15:34,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.6510 sec 2024-11-16T10:15:34,727 INFO [Thread-1071 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1071. 2024-11-16T10:15:34,727 DEBUG [Thread-1071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:15:34,727 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:34,727 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:34,727 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:15:34,728 INFO [Thread-1069 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1069. 2024-11-16T10:15:34,728 DEBUG [Thread-1069 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:15:34,728 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:34,728 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:34,728 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:15:34,728 INFO [Thread-1073 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1073. 2024-11-16T10:15:34,728 DEBUG [Thread-1073 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:15:34,728 DEBUG [Thread-1073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:34,729 DEBUG [Thread-1073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:34,729 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:15:34,730 INFO [Thread-1065 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1065. 2024-11-16T10:15:34,730 DEBUG [Thread-1065 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:15:34,730 DEBUG [Thread-1065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:34,730 DEBUG [Thread-1065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:34,730 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:15:34,731 INFO [Thread-1067 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1067. 2024-11-16T10:15:34,731 DEBUG [Thread-1067 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:15:34,731 DEBUG [Thread-1067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:34,731 DEBUG [Thread-1067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:34,731 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:15:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:34,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-16T10:15:34,865 INFO [Thread-1054 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1054. 2024-11-16T10:15:34,865 INFO [Thread-1060 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1060. 2024-11-16T10:15:34,865 DEBUG [Thread-1054 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:15:34,865 DEBUG [Thread-1060 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:15:34,865 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:34,865 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:34,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:34,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:34,865 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:34,865 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:34,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:34,865 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:15:34,865 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:15:34,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:34,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:34,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:34,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/049e17933f6f41f9a76e3856d38624ed is 50, key is test_row_0/A:col10/1731752133717/Put/seqid=0 2024-11-16T10:15:34,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742151_1327 (size=12301) 2024-11-16T10:15:35,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T10:15:35,025 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:15:35,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=833 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/049e17933f6f41f9a76e3856d38624ed 2024-11-16T10:15:35,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/aa0bbd0d4e2c4c2d99c010f4ff3e7ccc is 50, key is test_row_0/B:col10/1731752133717/Put/seqid=0 2024-11-16T10:15:35,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742152_1328 (size=12301) 2024-11-16T10:15:35,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=833 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/aa0bbd0d4e2c4c2d99c010f4ff3e7ccc 2024-11-16T10:15:35,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/bbf2b460301241208131a02e286dbc62 is 50, key is test_row_0/C:col10/1731752133717/Put/seqid=0 2024-11-16T10:15:35,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742153_1329 (size=12301) 2024-11-16T10:15:35,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=833 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/bbf2b460301241208131a02e286dbc62 2024-11-16T10:15:35,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/049e17933f6f41f9a76e3856d38624ed as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/049e17933f6f41f9a76e3856d38624ed 2024-11-16T10:15:35,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/049e17933f6f41f9a76e3856d38624ed, entries=150, sequenceid=833, filesize=12.0 K 2024-11-16T10:15:35,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/aa0bbd0d4e2c4c2d99c010f4ff3e7ccc as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/aa0bbd0d4e2c4c2d99c010f4ff3e7ccc 2024-11-16T10:15:35,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/aa0bbd0d4e2c4c2d99c010f4ff3e7ccc, entries=150, sequenceid=833, filesize=12.0 K 2024-11-16T10:15:35,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/bbf2b460301241208131a02e286dbc62 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/bbf2b460301241208131a02e286dbc62 2024-11-16T10:15:35,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/bbf2b460301241208131a02e286dbc62, entries=150, sequenceid=833, filesize=12.0 K 2024-11-16T10:15:35,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for dcee08b7bd0f481f30af9d9b29c726c5 in 847ms, sequenceid=833, compaction requested=true 2024-11-16T10:15:35,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:35,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:15:35,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:35,712 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:15:35,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:15:35,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:35,712 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:15:35,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcee08b7bd0f481f30af9d9b29c726c5:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:15:35,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:35,713 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:15:35,713 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:15:35,713 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/A is initiating minor compaction (all files) 2024-11-16T10:15:35,713 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/B is initiating minor compaction (all files) 2024-11-16T10:15:35,713 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/A in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:35,713 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/B in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:35,713 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/6ef493cf24b04424a899b7e53b92de91, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/93e1d9a3e6ea4521acb8aeec94a0c870, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/bc86c67cbaec4d04b17d1ffb1e2fa0f3, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/049e17933f6f41f9a76e3856d38624ed] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=49.1 K 2024-11-16T10:15:35,713 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/531362762a0f441fa76fefb83ef9464a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/eab06b7ca5944051a29d33536be52a8a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/2f2eb6ae3d844ff59c78b7a0d02b8577, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/aa0bbd0d4e2c4c2d99c010f4ff3e7ccc] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=49.1 K 2024-11-16T10:15:35,714 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 531362762a0f441fa76fefb83ef9464a, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=783, earliestPutTs=1731752131682 2024-11-16T10:15:35,714 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ef493cf24b04424a899b7e53b92de91, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=783, earliestPutTs=1731752131682 2024-11-16T10:15:35,714 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting eab06b7ca5944051a29d33536be52a8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=796, earliestPutTs=1731752132876 2024-11-16T10:15:35,714 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93e1d9a3e6ea4521acb8aeec94a0c870, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=796, earliestPutTs=1731752132876 2024-11-16T10:15:35,714 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f2eb6ae3d844ff59c78b7a0d02b8577, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=818, earliestPutTs=1731752133055 2024-11-16T10:15:35,714 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc86c67cbaec4d04b17d1ffb1e2fa0f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=818, earliestPutTs=1731752133055 2024-11-16T10:15:35,714 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting aa0bbd0d4e2c4c2d99c010f4ff3e7ccc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=833, earliestPutTs=1731752133717 2024-11-16T10:15:35,714 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 049e17933f6f41f9a76e3856d38624ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=833, earliestPutTs=1731752133717 2024-11-16T10:15:35,722 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#B#compaction#289 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:35,722 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#A#compaction#288 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:35,722 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/3ce008a6979b497aae4161b7f35a9f9c is 50, key is test_row_0/B:col10/1731752133717/Put/seqid=0 2024-11-16T10:15:35,722 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/fee76d75b5f64b28b25787ee697e3bba is 50, key is test_row_0/A:col10/1731752133717/Put/seqid=0 2024-11-16T10:15:35,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742154_1330 (size=13493) 2024-11-16T10:15:35,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742155_1331 (size=13493) 2024-11-16T10:15:36,130 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/fee76d75b5f64b28b25787ee697e3bba as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/fee76d75b5f64b28b25787ee697e3bba 2024-11-16T10:15:36,130 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/3ce008a6979b497aae4161b7f35a9f9c as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/3ce008a6979b497aae4161b7f35a9f9c 2024-11-16T10:15:36,134 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/B of dcee08b7bd0f481f30af9d9b29c726c5 into 3ce008a6979b497aae4161b7f35a9f9c(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:36,134 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/A of dcee08b7bd0f481f30af9d9b29c726c5 into fee76d75b5f64b28b25787ee697e3bba(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:36,134 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:36,134 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:36,134 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/A, priority=12, startTime=1731752135712; duration=0sec 2024-11-16T10:15:36,134 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/B, priority=12, startTime=1731752135712; duration=0sec 2024-11-16T10:15:36,134 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:36,134 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:B 2024-11-16T10:15:36,134 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:36,134 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:A 2024-11-16T10:15:36,134 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:15:36,135 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:15:36,135 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): dcee08b7bd0f481f30af9d9b29c726c5/C is initiating minor compaction (all files) 2024-11-16T10:15:36,135 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcee08b7bd0f481f30af9d9b29c726c5/C in TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:36,135 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/6ca6f708835c4956a42e1c5215be1188, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/6bc7b64537c342c196ec67d7e4e463f1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/f5d7b66f6d0e4344bcb1fc4a01b8c283, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/bbf2b460301241208131a02e286dbc62] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp, totalSize=49.1 K 2024-11-16T10:15:36,135 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ca6f708835c4956a42e1c5215be1188, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=783, earliestPutTs=1731752131682 2024-11-16T10:15:36,136 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bc7b64537c342c196ec67d7e4e463f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=796, earliestPutTs=1731752132876 2024-11-16T10:15:36,136 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting f5d7b66f6d0e4344bcb1fc4a01b8c283, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=818, earliestPutTs=1731752133055 2024-11-16T10:15:36,136 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting bbf2b460301241208131a02e286dbc62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=833, earliestPutTs=1731752133717 2024-11-16T10:15:36,143 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcee08b7bd0f481f30af9d9b29c726c5#C#compaction#290 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:36,144 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/b5a492faa8b44a388b076ec23bc5e739 is 50, key is test_row_0/C:col10/1731752133717/Put/seqid=0 2024-11-16T10:15:36,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742156_1332 (size=13493) 2024-11-16T10:15:36,553 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/b5a492faa8b44a388b076ec23bc5e739 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/b5a492faa8b44a388b076ec23bc5e739 2024-11-16T10:15:36,561 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcee08b7bd0f481f30af9d9b29c726c5/C of dcee08b7bd0f481f30af9d9b29c726c5 into b5a492faa8b44a388b076ec23bc5e739(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:36,561 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcee08b7bd0f481f30af9d9b29c726c5: 2024-11-16T10:15:36,561 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., storeName=dcee08b7bd0f481f30af9d9b29c726c5/C, priority=12, startTime=1731752135712; duration=0sec 2024-11-16T10:15:36,562 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:36,562 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcee08b7bd0f481f30af9d9b29c726c5:C 2024-11-16T10:15:37,027 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T10:15:41,866 INFO [Thread-1058 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1058. 2024-11-16T10:15:41,866 DEBUG [Thread-1058 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:15:41,866 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:41,866 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:41,866 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:15:41,916 INFO [Thread-1062 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1062. 2024-11-16T10:15:41,916 DEBUG [Thread-1062 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:15:41,916 DEBUG [Thread-1062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:41,916 DEBUG [Thread-1062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:41,916 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:15:41,926 INFO [Thread-1056 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1056. 2024-11-16T10:15:41,926 DEBUG [Thread-1056 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:15:41,926 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:41,926 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:41,926 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:15:41,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T10:15:41,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 292 2024-11-16T10:15:41,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-16T10:15:41,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-16T10:15:41,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 245 2024-11-16T10:15:41,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-16T10:15:41,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T10:15:41,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2502 2024-11-16T10:15:41,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2527 2024-11-16T10:15:41,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2494 2024-11-16T10:15:41,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2499 2024-11-16T10:15:41,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2528 2024-11-16T10:15:41,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T10:15:41,926 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T10:15:41,926 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T10:15:41,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:41,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:41,927 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:15:41,927 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T10:15:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T10:15:41,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-16T10:15:41,931 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752141931"}]},"ts":"1731752141931"} 2024-11-16T10:15:41,932 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T10:15:41,932 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T10:15:41,933 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T10:15:41,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dcee08b7bd0f481f30af9d9b29c726c5, UNASSIGN}] 2024-11-16T10:15:41,935 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dcee08b7bd0f481f30af9d9b29c726c5, UNASSIGN 2024-11-16T10:15:41,936 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=dcee08b7bd0f481f30af9d9b29c726c5, regionState=CLOSING, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:41,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dcee08b7bd0f481f30af9d9b29c726c5, UNASSIGN because future has completed 2024-11-16T10:15:41,937 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T10:15:41,937 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912}] 2024-11-16T10:15:42,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-16T10:15:42,090 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:42,090 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T10:15:42,090 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing dcee08b7bd0f481f30af9d9b29c726c5, disabling compactions & flushes 2024-11-16T10:15:42,090 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:42,090 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:42,091 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. after waiting 0 ms 2024-11-16T10:15:42,091 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:42,091 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing dcee08b7bd0f481f30af9d9b29c726c5 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-16T10:15:42,091 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=A 2024-11-16T10:15:42,091 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:42,091 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=B 2024-11-16T10:15:42,091 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:42,091 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcee08b7bd0f481f30af9d9b29c726c5, store=C 2024-11-16T10:15:42,092 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:42,098 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/88a4ce63060041c0be9d628307130154 is 50, key is test_row_0/A:col10/1731752141865/Put/seqid=0 2024-11-16T10:15:42,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742157_1333 (size=12301) 2024-11-16T10:15:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-16T10:15:42,502 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=842 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/88a4ce63060041c0be9d628307130154 2024-11-16T10:15:42,510 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/336becdbe678420382ccc58f2ff0e7d9 is 50, key is test_row_0/B:col10/1731752141865/Put/seqid=0 2024-11-16T10:15:42,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742158_1334 (size=12301) 2024-11-16T10:15:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-16T10:15:42,916 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=842 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/336becdbe678420382ccc58f2ff0e7d9 2024-11-16T10:15:42,931 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/49f013af453e47b3820dfb1800a0c05f is 50, key is test_row_0/C:col10/1731752141865/Put/seqid=0 2024-11-16T10:15:42,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742159_1335 (size=12301) 2024-11-16T10:15:43,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-16T10:15:43,336 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=842 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/49f013af453e47b3820dfb1800a0c05f 2024-11-16T10:15:43,346 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/A/88a4ce63060041c0be9d628307130154 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/88a4ce63060041c0be9d628307130154 2024-11-16T10:15:43,352 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/88a4ce63060041c0be9d628307130154, entries=150, sequenceid=842, filesize=12.0 K 2024-11-16T10:15:43,353 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/B/336becdbe678420382ccc58f2ff0e7d9 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/336becdbe678420382ccc58f2ff0e7d9 2024-11-16T10:15:43,357 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/336becdbe678420382ccc58f2ff0e7d9, entries=150, sequenceid=842, filesize=12.0 K 2024-11-16T10:15:43,357 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/.tmp/C/49f013af453e47b3820dfb1800a0c05f as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/49f013af453e47b3820dfb1800a0c05f 2024-11-16T10:15:43,361 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/49f013af453e47b3820dfb1800a0c05f, entries=150, sequenceid=842, filesize=12.0 K 2024-11-16T10:15:43,362 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for dcee08b7bd0f481f30af9d9b29c726c5 in 1271ms, sequenceid=842, compaction requested=false 2024-11-16T10:15:43,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/437b4e9e29b74a4e91eb7bcc9a069077, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/b4c741b3a4184aee937779a40608f357, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/49ae7ced34c04c319671477089fe8cdc, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/0dc00fb0eab247059285376cc89fd131, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/38561af7b7b246c1a5a9e2f150611265, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/93fe9ba1f87240b3a107839e8850c72a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/da21fe390bc44a84ac89b8a74eca2f70, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/de5dfec7b96946ef80ec5950323f4a19, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/627750da63b842bc8d77d790dc877409, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/859b53a043b14ab4b43d0dfa75d6d1c5, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/c36586498d5341949c635c911ca87c21, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/89ea2b1096334e4e9f35badfb96bcb45, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/3135512089f94d1b8ce57a0a9c1cd66d, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/ca9905a026fb4e18a97444f87d9a0870, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/807b40800ec34867a9bb67f14a392b6e, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/5e8673ca927e4cb993309134e4958043, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/febad6b4d1e0468d85c1e74f1b78d49b, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/be98628ac57c4917a5eb5b25497ad993, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/0d768904bd0a4093acddee017ddb33d2, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/0b4e1e2281534eecb70a3ed476a68247, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/efc46ce7ef074cd598848bf8b5e5ab0b, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/49f47478a4b7453ebacfa8693d9f23a6, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/a8b56b157eb541ffbdec3130091f7d67, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/9df5e2a91bb94fb9a023ac7e52e7b5cc, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/325d66fb84ec4554af167022abc67cba, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/9261f56acd9a4af09abef4ad9e9a0250, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/a4f2489b72134d798f624ddcfaadd85e, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/a3574d5aee4540ea94e76e9adb9909f2, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/231ea934053a40088b9bce038164fc78, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/3dcfaedeb98044afa8232a9294e7cd65, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/4c7f6644236144a3b2007036d886b83b, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/6ef493cf24b04424a899b7e53b92de91, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/93e1d9a3e6ea4521acb8aeec94a0c870, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/bc86c67cbaec4d04b17d1ffb1e2fa0f3, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/049e17933f6f41f9a76e3856d38624ed] to archive 2024-11-16T10:15:43,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:15:43,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/437b4e9e29b74a4e91eb7bcc9a069077 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/437b4e9e29b74a4e91eb7bcc9a069077 2024-11-16T10:15:43,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/b4c741b3a4184aee937779a40608f357 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/b4c741b3a4184aee937779a40608f357 2024-11-16T10:15:43,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/49ae7ced34c04c319671477089fe8cdc to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/49ae7ced34c04c319671477089fe8cdc 2024-11-16T10:15:43,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/0dc00fb0eab247059285376cc89fd131 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/0dc00fb0eab247059285376cc89fd131 2024-11-16T10:15:43,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/38561af7b7b246c1a5a9e2f150611265 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/38561af7b7b246c1a5a9e2f150611265 2024-11-16T10:15:43,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/93fe9ba1f87240b3a107839e8850c72a to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/93fe9ba1f87240b3a107839e8850c72a 2024-11-16T10:15:43,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/da21fe390bc44a84ac89b8a74eca2f70 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/da21fe390bc44a84ac89b8a74eca2f70 2024-11-16T10:15:43,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/de5dfec7b96946ef80ec5950323f4a19 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/de5dfec7b96946ef80ec5950323f4a19 2024-11-16T10:15:43,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/627750da63b842bc8d77d790dc877409 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/627750da63b842bc8d77d790dc877409 2024-11-16T10:15:43,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/859b53a043b14ab4b43d0dfa75d6d1c5 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/859b53a043b14ab4b43d0dfa75d6d1c5 2024-11-16T10:15:43,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/c36586498d5341949c635c911ca87c21 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/c36586498d5341949c635c911ca87c21 2024-11-16T10:15:43,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/89ea2b1096334e4e9f35badfb96bcb45 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/89ea2b1096334e4e9f35badfb96bcb45 2024-11-16T10:15:43,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/3135512089f94d1b8ce57a0a9c1cd66d to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/3135512089f94d1b8ce57a0a9c1cd66d 2024-11-16T10:15:43,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/ca9905a026fb4e18a97444f87d9a0870 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/ca9905a026fb4e18a97444f87d9a0870 2024-11-16T10:15:43,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/807b40800ec34867a9bb67f14a392b6e to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/807b40800ec34867a9bb67f14a392b6e 2024-11-16T10:15:43,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/5e8673ca927e4cb993309134e4958043 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/5e8673ca927e4cb993309134e4958043 2024-11-16T10:15:43,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/febad6b4d1e0468d85c1e74f1b78d49b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/febad6b4d1e0468d85c1e74f1b78d49b 2024-11-16T10:15:43,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/be98628ac57c4917a5eb5b25497ad993 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/be98628ac57c4917a5eb5b25497ad993 2024-11-16T10:15:43,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/0d768904bd0a4093acddee017ddb33d2 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/0d768904bd0a4093acddee017ddb33d2 2024-11-16T10:15:43,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/0b4e1e2281534eecb70a3ed476a68247 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/0b4e1e2281534eecb70a3ed476a68247 2024-11-16T10:15:43,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/efc46ce7ef074cd598848bf8b5e5ab0b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/efc46ce7ef074cd598848bf8b5e5ab0b 2024-11-16T10:15:43,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/49f47478a4b7453ebacfa8693d9f23a6 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/49f47478a4b7453ebacfa8693d9f23a6 2024-11-16T10:15:43,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/a8b56b157eb541ffbdec3130091f7d67 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/a8b56b157eb541ffbdec3130091f7d67 2024-11-16T10:15:43,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/9df5e2a91bb94fb9a023ac7e52e7b5cc to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/9df5e2a91bb94fb9a023ac7e52e7b5cc 2024-11-16T10:15:43,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/325d66fb84ec4554af167022abc67cba to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/325d66fb84ec4554af167022abc67cba 2024-11-16T10:15:43,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/9261f56acd9a4af09abef4ad9e9a0250 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/9261f56acd9a4af09abef4ad9e9a0250 2024-11-16T10:15:43,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/a4f2489b72134d798f624ddcfaadd85e to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/a4f2489b72134d798f624ddcfaadd85e 2024-11-16T10:15:43,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/a3574d5aee4540ea94e76e9adb9909f2 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/a3574d5aee4540ea94e76e9adb9909f2 2024-11-16T10:15:43,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/231ea934053a40088b9bce038164fc78 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/231ea934053a40088b9bce038164fc78 2024-11-16T10:15:43,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/3dcfaedeb98044afa8232a9294e7cd65 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/3dcfaedeb98044afa8232a9294e7cd65 2024-11-16T10:15:43,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/4c7f6644236144a3b2007036d886b83b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/4c7f6644236144a3b2007036d886b83b 2024-11-16T10:15:43,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/6ef493cf24b04424a899b7e53b92de91 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/6ef493cf24b04424a899b7e53b92de91 2024-11-16T10:15:43,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/93e1d9a3e6ea4521acb8aeec94a0c870 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/93e1d9a3e6ea4521acb8aeec94a0c870 2024-11-16T10:15:43,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/bc86c67cbaec4d04b17d1ffb1e2fa0f3 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/bc86c67cbaec4d04b17d1ffb1e2fa0f3 2024-11-16T10:15:43,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/049e17933f6f41f9a76e3856d38624ed to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/049e17933f6f41f9a76e3856d38624ed 2024-11-16T10:15:43,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/455a0c5270034b9393309a19af9309db, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/03eecf08b0e64c9d950f23da87559707, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/b9a511fb68f94a79b6865677396f0e9f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/8a8833fa522149408d59599923aee088, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/a2f42fd7d34f47ecb6c7bf7ef24079da, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/43a74bdf91604bd2b741bdf99b28201a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/d8fea82f00e44ff0a666d1b5a3145d54, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/b4867d784dce44c9a6253ed3dbd31476, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/d9a98c27e5844a958f3cbe78e5a922d2, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/ec7a274901354bad8e00fb71c6866c9d, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/bc535bea1553454bafe3248b8b484457, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/d86f79cdee2e45d593ea7c25948ede97, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/89575ef3838d495998d8494c7432b7ef, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/64c948eaee2a4a50b83a1c4af2efc6a3, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/d4e60aae835b4d42bb21a2f5fad19274, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/23666c9856494979b4ede6dba94febae, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/3c3b22bc8dae467287d9b513ab928250, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/14fb52410bff4213a6148de6bc2a8472, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/c673c28a377a4bbc9fe508fb9ae58a13, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/4a9c27db8a074be58dea07309154a66f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/a75bd5cef3e04bafae70c91032cd90e5, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/015ef4c24bfc46ba9e8df419163b3f46, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/177a25ffef5542b1a2d5e3a4762e17a8, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/959ef2b1bbdb4ec4a492af26f10be3c7, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/f66ae675f53a4f91b0055411b4627877, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/fd373d7553f3434d9baffc4c7c488622, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/aae16e284fa54aa5a1c2dd224db0e347, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/6e86be9fa154487aa34fd7f854fd93af, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/c95a1248d25d41c39dbdaf6f1fcb8cc4, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/c41ea5bb98004eba85a8c76d753a7d66, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/531362762a0f441fa76fefb83ef9464a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/575745ce818e4383a2ed7fa2ecab9c34, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/eab06b7ca5944051a29d33536be52a8a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/2f2eb6ae3d844ff59c78b7a0d02b8577, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/aa0bbd0d4e2c4c2d99c010f4ff3e7ccc] to archive 2024-11-16T10:15:43,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:15:43,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/455a0c5270034b9393309a19af9309db to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/455a0c5270034b9393309a19af9309db 2024-11-16T10:15:43,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/03eecf08b0e64c9d950f23da87559707 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/03eecf08b0e64c9d950f23da87559707 2024-11-16T10:15:43,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/b9a511fb68f94a79b6865677396f0e9f to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/b9a511fb68f94a79b6865677396f0e9f 2024-11-16T10:15:43,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/8a8833fa522149408d59599923aee088 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/8a8833fa522149408d59599923aee088 2024-11-16T10:15:43,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/a2f42fd7d34f47ecb6c7bf7ef24079da to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/a2f42fd7d34f47ecb6c7bf7ef24079da 2024-11-16T10:15:43,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/43a74bdf91604bd2b741bdf99b28201a to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/43a74bdf91604bd2b741bdf99b28201a 2024-11-16T10:15:43,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/d8fea82f00e44ff0a666d1b5a3145d54 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/d8fea82f00e44ff0a666d1b5a3145d54 2024-11-16T10:15:43,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/b4867d784dce44c9a6253ed3dbd31476 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/b4867d784dce44c9a6253ed3dbd31476 2024-11-16T10:15:43,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/d9a98c27e5844a958f3cbe78e5a922d2 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/d9a98c27e5844a958f3cbe78e5a922d2 2024-11-16T10:15:43,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/ec7a274901354bad8e00fb71c6866c9d to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/ec7a274901354bad8e00fb71c6866c9d 2024-11-16T10:15:43,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/bc535bea1553454bafe3248b8b484457 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/bc535bea1553454bafe3248b8b484457 2024-11-16T10:15:43,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/d86f79cdee2e45d593ea7c25948ede97 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/d86f79cdee2e45d593ea7c25948ede97 2024-11-16T10:15:43,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/89575ef3838d495998d8494c7432b7ef to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/89575ef3838d495998d8494c7432b7ef 2024-11-16T10:15:43,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/64c948eaee2a4a50b83a1c4af2efc6a3 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/64c948eaee2a4a50b83a1c4af2efc6a3 2024-11-16T10:15:43,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/d4e60aae835b4d42bb21a2f5fad19274 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/d4e60aae835b4d42bb21a2f5fad19274 2024-11-16T10:15:43,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/23666c9856494979b4ede6dba94febae to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/23666c9856494979b4ede6dba94febae 2024-11-16T10:15:43,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/3c3b22bc8dae467287d9b513ab928250 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/3c3b22bc8dae467287d9b513ab928250 2024-11-16T10:15:43,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/14fb52410bff4213a6148de6bc2a8472 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/14fb52410bff4213a6148de6bc2a8472 2024-11-16T10:15:43,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/c673c28a377a4bbc9fe508fb9ae58a13 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/c673c28a377a4bbc9fe508fb9ae58a13 2024-11-16T10:15:43,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/4a9c27db8a074be58dea07309154a66f to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/4a9c27db8a074be58dea07309154a66f 2024-11-16T10:15:43,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/a75bd5cef3e04bafae70c91032cd90e5 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/a75bd5cef3e04bafae70c91032cd90e5 2024-11-16T10:15:43,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/015ef4c24bfc46ba9e8df419163b3f46 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/015ef4c24bfc46ba9e8df419163b3f46 2024-11-16T10:15:43,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/177a25ffef5542b1a2d5e3a4762e17a8 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/177a25ffef5542b1a2d5e3a4762e17a8 2024-11-16T10:15:43,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/959ef2b1bbdb4ec4a492af26f10be3c7 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/959ef2b1bbdb4ec4a492af26f10be3c7 2024-11-16T10:15:43,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/f66ae675f53a4f91b0055411b4627877 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/f66ae675f53a4f91b0055411b4627877 2024-11-16T10:15:43,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/fd373d7553f3434d9baffc4c7c488622 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/fd373d7553f3434d9baffc4c7c488622 2024-11-16T10:15:43,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/aae16e284fa54aa5a1c2dd224db0e347 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/aae16e284fa54aa5a1c2dd224db0e347 2024-11-16T10:15:43,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/6e86be9fa154487aa34fd7f854fd93af to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/6e86be9fa154487aa34fd7f854fd93af 2024-11-16T10:15:43,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/c95a1248d25d41c39dbdaf6f1fcb8cc4 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/c95a1248d25d41c39dbdaf6f1fcb8cc4 2024-11-16T10:15:43,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/c41ea5bb98004eba85a8c76d753a7d66 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/c41ea5bb98004eba85a8c76d753a7d66 2024-11-16T10:15:43,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/531362762a0f441fa76fefb83ef9464a to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/531362762a0f441fa76fefb83ef9464a 2024-11-16T10:15:43,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/575745ce818e4383a2ed7fa2ecab9c34 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/575745ce818e4383a2ed7fa2ecab9c34 2024-11-16T10:15:43,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/eab06b7ca5944051a29d33536be52a8a to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/eab06b7ca5944051a29d33536be52a8a 2024-11-16T10:15:43,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/2f2eb6ae3d844ff59c78b7a0d02b8577 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/2f2eb6ae3d844ff59c78b7a0d02b8577 2024-11-16T10:15:43,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/aa0bbd0d4e2c4c2d99c010f4ff3e7ccc to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/aa0bbd0d4e2c4c2d99c010f4ff3e7ccc 2024-11-16T10:15:43,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/e0ea2ebf7efe4aa5a442f246ac586947, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/f2b0398b2db1480497c0ab6f34204771, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/8e6663d15a63406ea10b7c0c0151fddf, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/09cc4ebfe7fa4f20b4b118f708e04aac, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/19e7fd80b46542858568d2b82ebd7aa6, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/12075f546ef74fa7813e373417f76e18, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/6f04a8393324424283f8f22f689f55f8, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/8bdb1aa1e3da4834b60d738b0588b49c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/bc1ba5f7a0064d0281c07234c95cd3cd, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/b9b537acd31543d4b8d776883b8063b4, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/98e94354a0c34e19976b8e00dd0a8a44, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/d5b766d686c14dc2bb93aee175b2b124, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/fd256d2f236f4b0d9427be71a8cd9af3, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/1e0d9e221d68437e8a2298c0aa743675, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/30ea12433f68464da3af7b8f2090a490, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/1c41b19cc02f403d9e748731ee1597df, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/e9a1a3a9f80d4e9ca31f52ecfc3c4840, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/869018c167f34f07ab12037f6f6db647, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/41f24033debb4578b53e01c3839f51ec, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/12d3c799df7b4a3f90a9a1f21f20b77c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/88f700f1b48446379ad8cd3b55202bf6, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/92785a21e15d432d89a0d3ac8e2b31c1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/af8a2d0f514843c7b8147e82c7ba218c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/c2687817c73d4b7495ce3d55f107a2f4, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/769fb5de49c34f7787cd90cef7debc3d, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/97dbb3d0559246b79c919f1fbc382d43, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/0b92dacba94b4b879806a152d94f95ef, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/79762ded297e4da3959b68a494cca8d3, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/166b5696520448e4bd373c59d05f33da, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/46e3dd01c1e1472ca8eb266664edda19, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/6ca6f708835c4956a42e1c5215be1188, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/f80bda994c574de8924cc91e667f1183, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/6bc7b64537c342c196ec67d7e4e463f1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/f5d7b66f6d0e4344bcb1fc4a01b8c283, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/bbf2b460301241208131a02e286dbc62] to archive 2024-11-16T10:15:43,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:15:43,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/e0ea2ebf7efe4aa5a442f246ac586947 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/e0ea2ebf7efe4aa5a442f246ac586947 2024-11-16T10:15:43,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/f2b0398b2db1480497c0ab6f34204771 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/f2b0398b2db1480497c0ab6f34204771 2024-11-16T10:15:43,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/8e6663d15a63406ea10b7c0c0151fddf to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/8e6663d15a63406ea10b7c0c0151fddf 2024-11-16T10:15:43,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/09cc4ebfe7fa4f20b4b118f708e04aac to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/09cc4ebfe7fa4f20b4b118f708e04aac 2024-11-16T10:15:43,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/19e7fd80b46542858568d2b82ebd7aa6 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/19e7fd80b46542858568d2b82ebd7aa6 2024-11-16T10:15:43,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/12075f546ef74fa7813e373417f76e18 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/12075f546ef74fa7813e373417f76e18 2024-11-16T10:15:43,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/6f04a8393324424283f8f22f689f55f8 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/6f04a8393324424283f8f22f689f55f8 2024-11-16T10:15:43,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/8bdb1aa1e3da4834b60d738b0588b49c to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/8bdb1aa1e3da4834b60d738b0588b49c 2024-11-16T10:15:43,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/bc1ba5f7a0064d0281c07234c95cd3cd to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/bc1ba5f7a0064d0281c07234c95cd3cd 2024-11-16T10:15:43,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/b9b537acd31543d4b8d776883b8063b4 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/b9b537acd31543d4b8d776883b8063b4 2024-11-16T10:15:43,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/98e94354a0c34e19976b8e00dd0a8a44 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/98e94354a0c34e19976b8e00dd0a8a44 2024-11-16T10:15:43,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/d5b766d686c14dc2bb93aee175b2b124 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/d5b766d686c14dc2bb93aee175b2b124 2024-11-16T10:15:43,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/fd256d2f236f4b0d9427be71a8cd9af3 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/fd256d2f236f4b0d9427be71a8cd9af3 2024-11-16T10:15:43,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/1e0d9e221d68437e8a2298c0aa743675 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/1e0d9e221d68437e8a2298c0aa743675 2024-11-16T10:15:43,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/30ea12433f68464da3af7b8f2090a490 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/30ea12433f68464da3af7b8f2090a490 2024-11-16T10:15:43,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/1c41b19cc02f403d9e748731ee1597df to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/1c41b19cc02f403d9e748731ee1597df 2024-11-16T10:15:43,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/e9a1a3a9f80d4e9ca31f52ecfc3c4840 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/e9a1a3a9f80d4e9ca31f52ecfc3c4840 2024-11-16T10:15:43,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/869018c167f34f07ab12037f6f6db647 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/869018c167f34f07ab12037f6f6db647 2024-11-16T10:15:43,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/41f24033debb4578b53e01c3839f51ec to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/41f24033debb4578b53e01c3839f51ec 2024-11-16T10:15:43,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/12d3c799df7b4a3f90a9a1f21f20b77c to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/12d3c799df7b4a3f90a9a1f21f20b77c 2024-11-16T10:15:43,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/88f700f1b48446379ad8cd3b55202bf6 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/88f700f1b48446379ad8cd3b55202bf6 2024-11-16T10:15:43,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/92785a21e15d432d89a0d3ac8e2b31c1 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/92785a21e15d432d89a0d3ac8e2b31c1 2024-11-16T10:15:43,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/af8a2d0f514843c7b8147e82c7ba218c to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/af8a2d0f514843c7b8147e82c7ba218c 2024-11-16T10:15:43,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/c2687817c73d4b7495ce3d55f107a2f4 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/c2687817c73d4b7495ce3d55f107a2f4 2024-11-16T10:15:43,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/769fb5de49c34f7787cd90cef7debc3d to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/769fb5de49c34f7787cd90cef7debc3d 2024-11-16T10:15:43,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/97dbb3d0559246b79c919f1fbc382d43 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/97dbb3d0559246b79c919f1fbc382d43 2024-11-16T10:15:43,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/0b92dacba94b4b879806a152d94f95ef to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/0b92dacba94b4b879806a152d94f95ef 2024-11-16T10:15:43,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/79762ded297e4da3959b68a494cca8d3 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/79762ded297e4da3959b68a494cca8d3 2024-11-16T10:15:43,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/166b5696520448e4bd373c59d05f33da to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/166b5696520448e4bd373c59d05f33da 2024-11-16T10:15:43,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/46e3dd01c1e1472ca8eb266664edda19 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/46e3dd01c1e1472ca8eb266664edda19 2024-11-16T10:15:43,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/6ca6f708835c4956a42e1c5215be1188 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/6ca6f708835c4956a42e1c5215be1188 2024-11-16T10:15:43,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/f80bda994c574de8924cc91e667f1183 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/f80bda994c574de8924cc91e667f1183 2024-11-16T10:15:43,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/6bc7b64537c342c196ec67d7e4e463f1 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/6bc7b64537c342c196ec67d7e4e463f1 2024-11-16T10:15:43,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/f5d7b66f6d0e4344bcb1fc4a01b8c283 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/f5d7b66f6d0e4344bcb1fc4a01b8c283 2024-11-16T10:15:43,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/bbf2b460301241208131a02e286dbc62 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/bbf2b460301241208131a02e286dbc62 2024-11-16T10:15:43,450 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/recovered.edits/845.seqid, newMaxSeqId=845, maxSeqId=1 2024-11-16T10:15:43,451 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. 2024-11-16T10:15:43,451 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for dcee08b7bd0f481f30af9d9b29c726c5: Waiting for close lock at 1731752142090Running coprocessor pre-close hooks at 1731752142090Disabling compacts and flushes for region at 1731752142090Disabling writes for close at 1731752142091 (+1 ms)Obtaining lock to block concurrent updates at 1731752142091Preparing flush snapshotting stores in dcee08b7bd0f481f30af9d9b29c726c5 at 1731752142091Finished memstore snapshotting TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731752142092 (+1 ms)Flushing stores of TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5. at 1731752142092Flushing dcee08b7bd0f481f30af9d9b29c726c5/A: creating writer at 1731752142093 (+1 ms)Flushing dcee08b7bd0f481f30af9d9b29c726c5/A: appending metadata at 1731752142097 (+4 ms)Flushing dcee08b7bd0f481f30af9d9b29c726c5/A: closing flushed file at 1731752142097Flushing dcee08b7bd0f481f30af9d9b29c726c5/B: creating writer at 1731752142507 (+410 ms)Flushing dcee08b7bd0f481f30af9d9b29c726c5/B: appending metadata at 1731752142509 (+2 ms)Flushing dcee08b7bd0f481f30af9d9b29c726c5/B: closing flushed file at 1731752142509Flushing dcee08b7bd0f481f30af9d9b29c726c5/C: creating writer at 1731752142927 (+418 ms)Flushing dcee08b7bd0f481f30af9d9b29c726c5/C: appending metadata at 1731752142930 (+3 ms)Flushing dcee08b7bd0f481f30af9d9b29c726c5/C: closing flushed file at 1731752142930Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@679d5dbb: reopening flushed file at 1731752143345 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b0be171: reopening flushed file at 1731752143352 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@281621d8: reopening flushed file at 1731752143357 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for dcee08b7bd0f481f30af9d9b29c726c5 in 1271ms, sequenceid=842, compaction requested=false at 1731752143362 (+5 ms)Writing region close event to WAL at 1731752143448 (+86 ms)Running coprocessor post-close hooks at 1731752143451 (+3 ms)Closed at 1731752143451 2024-11-16T10:15:43,454 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=dcee08b7bd0f481f30af9d9b29c726c5, regionState=CLOSED 2024-11-16T10:15:43,454 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:43,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 because future has completed 2024-11-16T10:15:43,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-16T10:15:43,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure dcee08b7bd0f481f30af9d9b29c726c5, server=b16ff38458c1,45417,1731752048912 in 1.5190 sec 2024-11-16T10:15:43,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-16T10:15:43,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dcee08b7bd0f481f30af9d9b29c726c5, UNASSIGN in 1.5220 sec 2024-11-16T10:15:43,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-16T10:15:43,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5260 sec 2024-11-16T10:15:43,462 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752143462"}]},"ts":"1731752143462"} 2024-11-16T10:15:43,463 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T10:15:43,463 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T10:15:43,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5370 sec 2024-11-16T10:15:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-16T10:15:44,085 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:15:44,086 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T10:15:44,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:15:44,089 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:15:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-16T10:15:44,090 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:15:44,092 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:44,094 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A, FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B, FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C, FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/recovered.edits] 2024-11-16T10:15:44,096 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/88a4ce63060041c0be9d628307130154 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/88a4ce63060041c0be9d628307130154 2024-11-16T10:15:44,098 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/fee76d75b5f64b28b25787ee697e3bba to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/A/fee76d75b5f64b28b25787ee697e3bba 2024-11-16T10:15:44,100 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/336becdbe678420382ccc58f2ff0e7d9 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/336becdbe678420382ccc58f2ff0e7d9 2024-11-16T10:15:44,101 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/3ce008a6979b497aae4161b7f35a9f9c to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/B/3ce008a6979b497aae4161b7f35a9f9c 2024-11-16T10:15:44,103 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/49f013af453e47b3820dfb1800a0c05f to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/49f013af453e47b3820dfb1800a0c05f 2024-11-16T10:15:44,104 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/b5a492faa8b44a388b076ec23bc5e739 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/C/b5a492faa8b44a388b076ec23bc5e739 2024-11-16T10:15:44,107 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/recovered.edits/845.seqid to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5/recovered.edits/845.seqid 2024-11-16T10:15:44,108 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/dcee08b7bd0f481f30af9d9b29c726c5 2024-11-16T10:15:44,108 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T10:15:44,111 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:15:44,115 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T10:15:44,117 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T10:15:44,118 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:15:44,118 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T10:15:44,119 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731752144118"}]},"ts":"9223372036854775807"} 2024-11-16T10:15:44,121 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T10:15:44,121 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => dcee08b7bd0f481f30af9d9b29c726c5, NAME => 'TestAcidGuarantees,,1731752112404.dcee08b7bd0f481f30af9d9b29c726c5.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T10:15:44,121 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T10:15:44,121 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731752144121"}]},"ts":"9223372036854775807"} 2024-11-16T10:15:44,123 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T10:15:44,124 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:15:44,125 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 38 msec 2024-11-16T10:15:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-16T10:15:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T10:15:44,195 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:15:44,202 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 251), OpenFileDescriptor=453 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=406 (was 312) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4213 (was 3799) - AvailableMemoryMB LEAK? - 2024-11-16T10:15:44,210 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=406, ProcessCount=11, AvailableMemoryMB=4212 2024-11-16T10:15:44,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T10:15:44,212 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T10:15:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T10:15:44,214 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T10:15:44,214 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:44,215 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-11-16T10:15:44,215 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T10:15:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-16T10:15:44,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742160_1336 (size=1061) 2024-11-16T10:15:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-16T10:15:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-16T10:15:44,625 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0 2024-11-16T10:15:44,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742161_1337 (size=53) 2024-11-16T10:15:44,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-16T10:15:45,035 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:15:45,035 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6534162d1894b79648a1282b0163350b, disabling compactions & flushes 2024-11-16T10:15:45,035 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:45,035 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:45,035 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. after waiting 0 ms 2024-11-16T10:15:45,035 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:45,035 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:45,035 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6534162d1894b79648a1282b0163350b: Waiting for close lock at 1731752145035Disabling compacts and flushes for region at 1731752145035Disabling writes for close at 1731752145035Writing region close event to WAL at 1731752145035Closed at 1731752145035 2024-11-16T10:15:45,036 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T10:15:45,037 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731752145036"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731752145036"}]},"ts":"1731752145036"} 2024-11-16T10:15:45,038 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T10:15:45,039 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T10:15:45,039 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752145039"}]},"ts":"1731752145039"} 2024-11-16T10:15:45,040 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T10:15:45,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6534162d1894b79648a1282b0163350b, ASSIGN}] 2024-11-16T10:15:45,041 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6534162d1894b79648a1282b0163350b, ASSIGN 2024-11-16T10:15:45,042 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6534162d1894b79648a1282b0163350b, ASSIGN; state=OFFLINE, location=b16ff38458c1,45417,1731752048912; forceNewPlan=false, retain=false 2024-11-16T10:15:45,192 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=6534162d1894b79648a1282b0163350b, regionState=OPENING, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:45,194 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6534162d1894b79648a1282b0163350b, ASSIGN because future has completed 2024-11-16T10:15:45,194 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912}] 2024-11-16T10:15:45,351 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:45,351 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:15:45,352 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:45,352 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:15:45,352 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:45,352 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:45,353 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:45,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-16T10:15:45,354 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:15:45,354 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6534162d1894b79648a1282b0163350b columnFamilyName A 2024-11-16T10:15:45,355 DEBUG [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:45,355 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] regionserver.HStore(327): Store=6534162d1894b79648a1282b0163350b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:15:45,355 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:45,356 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:15:45,356 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6534162d1894b79648a1282b0163350b columnFamilyName B 2024-11-16T10:15:45,356 DEBUG [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:45,356 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] regionserver.HStore(327): Store=6534162d1894b79648a1282b0163350b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:15:45,357 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:45,357 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:15:45,358 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6534162d1894b79648a1282b0163350b columnFamilyName C 2024-11-16T10:15:45,358 DEBUG [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:45,358 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] regionserver.HStore(327): Store=6534162d1894b79648a1282b0163350b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:15:45,358 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:45,358 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:45,359 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b 2024-11-16T10:15:45,359 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b 2024-11-16T10:15:45,360 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:45,360 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:45,360 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:15:45,361 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:45,363 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T10:15:45,363 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened 6534162d1894b79648a1282b0163350b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61282427, jitterRate=-0.08682067692279816}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:15:45,364 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:45,364 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for 6534162d1894b79648a1282b0163350b: Running coprocessor pre-open hook at 1731752145352Writing region info on filesystem at 1731752145352Initializing all the Stores at 1731752145353 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752145353Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752145353Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752145353Cleaning up temporary data from old regions at 1731752145360 (+7 ms)Running coprocessor post-open hooks at 1731752145364 (+4 ms)Region opened successfully at 1731752145364 2024-11-16T10:15:45,365 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., pid=91, masterSystemTime=1731752145346 2024-11-16T10:15:45,367 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:45,367 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:45,367 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=6534162d1894b79648a1282b0163350b, regionState=OPEN, openSeqNum=2, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:45,369 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 because future has completed 2024-11-16T10:15:45,372 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-16T10:15:45,372 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 in 176 msec 2024-11-16T10:15:45,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-16T10:15:45,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6534162d1894b79648a1282b0163350b, ASSIGN in 332 msec 2024-11-16T10:15:45,375 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T10:15:45,375 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752145375"}]},"ts":"1731752145375"} 2024-11-16T10:15:45,376 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T10:15:45,377 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T10:15:45,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-11-16T10:15:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-16T10:15:46,364 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:15:46,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7befe57f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:46,366 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:15:46,366 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:15:46,368 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:15:46,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:15:46,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:15:46,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15448242, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:46,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:15:46,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:15:46,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:46,371 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:15:46,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@332c7c1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:46,373 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:15:46,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:15:46,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:15:46,375 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:15:46,376 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T10:15:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is b16ff38458c1,45949,1731752047888 2024-11-16T10:15:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@9f2d420 2024-11-16T10:15:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T10:15:46,379 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T10:15:46,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T10:15:46,381 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T10:15:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T10:15:46,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742162_1338 (size=1097) 2024-11-16T10:15:46,794 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-16T10:15:46,794 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-16T10:15:46,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T10:15:46,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6534162d1894b79648a1282b0163350b, REOPEN/MOVE}] 2024-11-16T10:15:46,798 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6534162d1894b79648a1282b0163350b, REOPEN/MOVE 2024-11-16T10:15:46,798 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=6534162d1894b79648a1282b0163350b, regionState=CLOSING, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:46,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6534162d1894b79648a1282b0163350b, REOPEN/MOVE because future has completed 2024-11-16T10:15:46,800 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T10:15:46,800 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912}] 2024-11-16T10:15:46,952 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:46,952 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T10:15:46,952 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing 6534162d1894b79648a1282b0163350b, disabling compactions & flushes 2024-11-16T10:15:46,952 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:46,952 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:46,952 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. after waiting 0 ms 2024-11-16T10:15:46,952 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:46,956 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T10:15:46,956 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:46,956 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for 6534162d1894b79648a1282b0163350b: Waiting for close lock at 1731752146952Running coprocessor pre-close hooks at 1731752146952Disabling compacts and flushes for region at 1731752146952Disabling writes for close at 1731752146952Writing region close event to WAL at 1731752146953 (+1 ms)Running coprocessor post-close hooks at 1731752146956 (+3 ms)Closed at 1731752146956 2024-11-16T10:15:46,956 WARN [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: 6534162d1894b79648a1282b0163350b to self. 2024-11-16T10:15:46,958 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:46,958 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=6534162d1894b79648a1282b0163350b, regionState=CLOSED 2024-11-16T10:15:46,959 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 because future has completed 2024-11-16T10:15:46,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-16T10:15:46,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure 6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 in 160 msec 2024-11-16T10:15:46,961 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6534162d1894b79648a1282b0163350b, REOPEN/MOVE; state=CLOSED, location=b16ff38458c1,45417,1731752048912; forceNewPlan=false, retain=true 2024-11-16T10:15:47,112 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=6534162d1894b79648a1282b0163350b, regionState=OPENING, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:47,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6534162d1894b79648a1282b0163350b, REOPEN/MOVE because future has completed 2024-11-16T10:15:47,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912}] 2024-11-16T10:15:47,273 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:47,273 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:15:47,274 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:47,274 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:15:47,274 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:47,274 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:47,275 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:47,276 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:15:47,276 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6534162d1894b79648a1282b0163350b columnFamilyName A 2024-11-16T10:15:47,277 DEBUG [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:47,278 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] regionserver.HStore(327): Store=6534162d1894b79648a1282b0163350b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:15:47,278 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:47,279 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:15:47,279 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6534162d1894b79648a1282b0163350b columnFamilyName B 2024-11-16T10:15:47,279 DEBUG [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:47,279 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] regionserver.HStore(327): Store=6534162d1894b79648a1282b0163350b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:15:47,279 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:47,280 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:15:47,280 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6534162d1894b79648a1282b0163350b columnFamilyName C 2024-11-16T10:15:47,280 DEBUG [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:47,280 INFO [StoreOpener-6534162d1894b79648a1282b0163350b-1 {}] regionserver.HStore(327): Store=6534162d1894b79648a1282b0163350b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:15:47,280 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:47,280 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:47,281 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b 2024-11-16T10:15:47,282 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b 2024-11-16T10:15:47,282 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:47,283 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:47,283 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:15:47,284 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:47,284 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened 6534162d1894b79648a1282b0163350b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66952366, jitterRate=-0.0023320019245147705}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:15:47,285 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:47,285 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for 6534162d1894b79648a1282b0163350b: Running coprocessor pre-open hook at 1731752147274Writing region info on filesystem at 1731752147274Initializing all the Stores at 1731752147275 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752147275Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752147275Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752147275Cleaning up temporary data from old regions at 1731752147283 (+8 ms)Running coprocessor post-open hooks at 1731752147285 (+2 ms)Region opened successfully at 1731752147285 2024-11-16T10:15:47,286 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., pid=96, masterSystemTime=1731752147270 2024-11-16T10:15:47,287 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:47,288 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:47,288 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=6534162d1894b79648a1282b0163350b, regionState=OPEN, openSeqNum=5, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:47,289 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 because future has completed 2024-11-16T10:15:47,292 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-11-16T10:15:47,292 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 in 174 msec 2024-11-16T10:15:47,301 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-16T10:15:47,301 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6534162d1894b79648a1282b0163350b, REOPEN/MOVE in 495 msec 2024-11-16T10:15:47,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-16T10:15:47,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 507 msec 2024-11-16T10:15:47,306 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 924 msec 2024-11-16T10:15:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-16T10:15:47,308 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:15:47,309 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,309 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:15:47,310 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:15:47,311 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:15:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:15:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:15:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d8735c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:15:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:15:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:47,313 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:15:47,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7d0943, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,318 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:15:47,318 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:15:47,319 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:15:47,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:15:47,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:15:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@421549b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:15:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:15:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:47,321 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:15:47,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@115f2f1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,322 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:15:47,322 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:15:47,323 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:15:47,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:15:47,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:15:47,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74445444, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:15:47,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:15:47,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:47,324 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:15:47,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f96beaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,325 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:15:47,325 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:15:47,326 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:15:47,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:15:47,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:15:47,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59fa60d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:15:47,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:15:47,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:47,328 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:15:47,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e4e081d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f004f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,329 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:15:47,329 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:15:47,330 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:15:47,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:15:47,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:15:47,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd890f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:15:47,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:15:47,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:47,331 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:15:47,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@542cb99b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,332 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:15:47,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:15:47,333 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:15:47,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:15:47,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:15:47,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3260565a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:15:47,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:15:47,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:47,334 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:15:47,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bbd0401, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,335 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:15:47,335 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:15:47,336 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:15:47,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:15:47,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:15:47,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a210db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:15:47,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:15:47,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:47,337 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:15:47,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57f19dc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,337 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:15:47,337 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:15:47,338 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:15:47,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:15:47,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:15:47,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cf3b2b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:15:47,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:15:47,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:47,339 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:15:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb23b86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,340 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:15:47,340 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:15:47,341 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:15:47,341 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:15:47,341 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:15:47,341 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17f63200, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,341 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:15:47,341 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:15:47,341 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:47,342 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:15:47,342 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4177cb03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,343 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa71035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,343 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:15:47,343 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:15:47,344 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:15:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:15:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:15:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1f2291, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:15:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:15:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:15:47,345 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:15:47,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626f0eea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:15:47,347 DEBUG [Thread-1557 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:15:47,347 DEBUG [Thread-1559 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:15:47,347 DEBUG [Thread-1553 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:15:47,347 DEBUG [Thread-1544 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:15:47,347 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:15:47,347 DEBUG [Thread-1551 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:15:47,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:15:47,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:15:47,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:15:47,348 DEBUG [Thread-1542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:15:47,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:15:47,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:15:47,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-16T10:15:47,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:15:47,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:15:47,349 DEBUG [Thread-1546 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:15:47,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:15:47,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:15:47,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:15:47,349 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:15:47,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:15:47,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:15:47,350 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:15:47,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T10:15:47,350 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:15:47,350 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:15:47,350 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:15:47,350 DEBUG [Thread-1546 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:15:47,350 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:15:47,350 DEBUG [Thread-1546 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:15:47,351 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:15:47,351 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:15:47,351 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:15:47,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:15:47,352 DEBUG [Thread-1540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:15:47,352 DEBUG [Thread-1548 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:15:47,352 DEBUG [Thread-1555 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:15:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:15:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:15:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:15:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:15:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:15:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:15:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:15:47,354 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:15:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:15:47,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:15:47,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:15:47,355 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:15:47,355 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:15:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:15:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:15:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:15:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:15:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:15:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:15:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:47,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6534162d1894b79648a1282b0163350b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T10:15:47,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=A 2024-11-16T10:15:47,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:47,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=B 2024-11-16T10:15:47,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:47,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=C 2024-11-16T10:15:47,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:47,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f1caeb907246498ca89cd5c06026e209_6534162d1894b79648a1282b0163350b is 50, key is test_row_0/A:col10/1731752147364/Put/seqid=0 2024-11-16T10:15:47,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742163_1339 (size=14594) 2024-11-16T10:15:47,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752207396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:47,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:47,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752207399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:47,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752207402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:47,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:47,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752207403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:47,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:47,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752207405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T10:15:47,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:15:47,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:47,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:47,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:47,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:47,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:47,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752207514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:47,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752207514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:47,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752207514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:47,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:47,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:47,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:47,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752207524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:47,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752207524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:47,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:47,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:15:47,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:47,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:47,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:47,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:47,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T10:15:47,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752207724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:47,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:47,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752207734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:47,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752207734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:47,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752207744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:47,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752207745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:47,795 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:47,798 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f1caeb907246498ca89cd5c06026e209_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f1caeb907246498ca89cd5c06026e209_6534162d1894b79648a1282b0163350b 2024-11-16T10:15:47,802 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/e829c5a1bd2b4bea9f422d6833ec6aa0, store: [table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:47,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/e829c5a1bd2b4bea9f422d6833ec6aa0 is 175, key is test_row_0/A:col10/1731752147364/Put/seqid=0 2024-11-16T10:15:47,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:15:47,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:47,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:47,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:47,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:47,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:47,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742164_1340 (size=39549) 2024-11-16T10:15:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:47,811 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/e829c5a1bd2b4bea9f422d6833ec6aa0 2024-11-16T10:15:47,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/1c339bfbd4e84479b8509503c86b95cd is 50, key is test_row_0/B:col10/1731752147364/Put/seqid=0 2024-11-16T10:15:47,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742165_1341 (size=12001) 2024-11-16T10:15:47,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/1c339bfbd4e84479b8509503c86b95cd 2024-11-16T10:15:47,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/12e5403801704924b5792f46c6714808 is 50, key is test_row_0/C:col10/1731752147364/Put/seqid=0 2024-11-16T10:15:47,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742166_1342 (size=12001) 2024-11-16T10:15:47,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/12e5403801704924b5792f46c6714808 2024-11-16T10:15:47,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/e829c5a1bd2b4bea9f422d6833ec6aa0 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/e829c5a1bd2b4bea9f422d6833ec6aa0 2024-11-16T10:15:47,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/e829c5a1bd2b4bea9f422d6833ec6aa0, entries=200, sequenceid=16, filesize=38.6 K 2024-11-16T10:15:47,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/1c339bfbd4e84479b8509503c86b95cd as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/1c339bfbd4e84479b8509503c86b95cd 2024-11-16T10:15:47,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:15:47,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:47,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:47,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:47,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:47,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:47,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/1c339bfbd4e84479b8509503c86b95cd, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T10:15:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T10:15:47,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/12e5403801704924b5792f46c6714808 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/12e5403801704924b5792f46c6714808 2024-11-16T10:15:47,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/12e5403801704924b5792f46c6714808, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T10:15:47,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6534162d1894b79648a1282b0163350b in 631ms, sequenceid=16, compaction requested=false 2024-11-16T10:15:47,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:48,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752147401, max timestamp=1731752148037Num uniques 859; 2024-11-16T10:15:48,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752147401, max timestamp=1731752148037Num uniques 859; 2024-11-16T10:15:48,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:15:48,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:15:48,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752147401, max timestamp=1731752148037Num uniques 859; 2024-11-16T10:15:48,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:15:48,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:48,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6534162d1894b79648a1282b0163350b 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-16T10:15:48,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=A 2024-11-16T10:15:48,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:48,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=B 2024-11-16T10:15:48,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:48,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=C 2024-11-16T10:15:48,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:48,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111646cb0fce803d467688f3b3ff22e1b322_6534162d1894b79648a1282b0163350b is 50, key is test_row_0/A:col10/1731752148048/Put/seqid=0 2024-11-16T10:15:48,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742167_1343 (size=14594) 2024-11-16T10:15:48,088 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:48,091 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111646cb0fce803d467688f3b3ff22e1b322_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111646cb0fce803d467688f3b3ff22e1b322_6534162d1894b79648a1282b0163350b 2024-11-16T10:15:48,092 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/6c6d61ccc9a8457fb56a467f4a9de079, store: [table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:48,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/6c6d61ccc9a8457fb56a467f4a9de079 is 175, key is test_row_0/A:col10/1731752148048/Put/seqid=0 2024-11-16T10:15:48,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742168_1344 (size=39549) 2024-11-16T10:15:48,098 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/6c6d61ccc9a8457fb56a467f4a9de079 2024-11-16T10:15:48,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/3ac1ff9c083d4491bc84eb057c1f2a03 is 50, key is test_row_0/B:col10/1731752148048/Put/seqid=0 2024-11-16T10:15:48,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742169_1345 (size=12001) 2024-11-16T10:15:48,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/3ac1ff9c083d4491bc84eb057c1f2a03 2024-11-16T10:15:48,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:15:48,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:48,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:48,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:48,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:48,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:48,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731752148079, max timestamp=1731752148121Num uniques 859; 2024-11-16T10:15:48,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T10:15:48,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/106301c006ce453e9b20a62341abb688 is 50, key is test_row_0/C:col10/1731752148048/Put/seqid=0 2024-11-16T10:15:48,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752148079, max timestamp=1731752148121Num uniques 859; 2024-11-16T10:15:48,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:15:48,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752148079, max timestamp=1731752148121Num uniques 859; 2024-11-16T10:15:48,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:15:48,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742170_1346 (size=12001) 2024-11-16T10:15:48,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/106301c006ce453e9b20a62341abb688 2024-11-16T10:15:48,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/6c6d61ccc9a8457fb56a467f4a9de079 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/6c6d61ccc9a8457fb56a467f4a9de079 2024-11-16T10:15:48,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/6c6d61ccc9a8457fb56a467f4a9de079, entries=200, sequenceid=41, filesize=38.6 K 2024-11-16T10:15:48,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/3ac1ff9c083d4491bc84eb057c1f2a03 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/3ac1ff9c083d4491bc84eb057c1f2a03 2024-11-16T10:15:48,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/3ac1ff9c083d4491bc84eb057c1f2a03, entries=150, sequenceid=41, filesize=11.7 K 2024-11-16T10:15:48,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/106301c006ce453e9b20a62341abb688 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/106301c006ce453e9b20a62341abb688 2024-11-16T10:15:48,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/106301c006ce453e9b20a62341abb688, entries=150, sequenceid=41, filesize=11.7 K 2024-11-16T10:15:48,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=118.53 KB/121370 for 6534162d1894b79648a1282b0163350b in 140ms, sequenceid=41, compaction requested=false 2024-11-16T10:15:48,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:48,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6534162d1894b79648a1282b0163350b 3/3 column families, dataSize=125.23 KB heapSize=355.30 KB 2024-11-16T10:15:48,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=A 2024-11-16T10:15:48,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:48,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=B 2024-11-16T10:15:48,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:48,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=C 2024-11-16T10:15:48,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:48,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116cf9b7f53feb54a9f86dc0fa9f21d305a_6534162d1894b79648a1282b0163350b is 50, key is test_row_0/A:col10/1731752148192/Put/seqid=0 2024-11-16T10:15:48,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752208218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:48,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:48,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752208223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:48,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:48,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:48,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752208245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:48,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:48,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:15:48,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:48,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:48,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:48,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:48,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:48,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:48,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752208247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:48,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:48,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:48,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742171_1347 (size=21918) 2024-11-16T10:15:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752208249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:48,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:48,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:48,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752208354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:48,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:48,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:48,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752208364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:48,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:48,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752208364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:48,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:48,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:48,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752208384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:48,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:48,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752208384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:48,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:48,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:48,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:15:48,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:48,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:48,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:48,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:48,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:48,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T10:15:48,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:15:48,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:48,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:48,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:48,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:48,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:48,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:48,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:48,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752208574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:48,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:48,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752208585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:48,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:48,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752208585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:48,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:48,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752208594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:48,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752208594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:48,676 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:48,680 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116cf9b7f53feb54a9f86dc0fa9f21d305a_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116cf9b7f53feb54a9f86dc0fa9f21d305a_6534162d1894b79648a1282b0163350b 2024-11-16T10:15:48,680 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/f8d2e7a0906a461fb1ce3f9705218436, store: [table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:48,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/f8d2e7a0906a461fb1ce3f9705218436 is 175, key is test_row_0/A:col10/1731752148192/Put/seqid=0 2024-11-16T10:15:48,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742172_1348 (size=65323) 2024-11-16T10:15:48,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:15:48,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:48,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:48,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:48,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:48,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:48,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:48,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:15:48,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:48,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:48,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:48,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:48,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:48,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:48,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:48,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752208894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:48,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:48,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752208894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:48,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:48,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752208895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:48,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752208914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:48,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:48,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752208914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:48,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:48,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:48,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:48,985 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T10:15:49,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:15:49,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:49,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:49,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:49,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:49,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:49,086 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/f8d2e7a0906a461fb1ce3f9705218436 2024-11-16T10:15:49,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/12a8133be48841a4bb5591c4eac582b1 is 50, key is test_row_0/B:col10/1731752148192/Put/seqid=0 2024-11-16T10:15:49,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742173_1349 (size=12001) 2024-11-16T10:15:49,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:15:49,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:49,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:49,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:49,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:49,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:49,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:49,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:15:49,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:49,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:49,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:49,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:49,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:49,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:49,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752209414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:49,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:49,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:49,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752209414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:49,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:49,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752209414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:49,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752209434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:49,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:49,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:49,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752209434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:49,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:49,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:49,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:49,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:49,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T10:15:49,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:15:49,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:49,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:49,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:49,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:49,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:49,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/12a8133be48841a4bb5591c4eac582b1 2024-11-16T10:15:49,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/aec028b42e1b47c89d2e480d686a7727 is 50, key is test_row_0/C:col10/1731752148192/Put/seqid=0 2024-11-16T10:15:49,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742174_1350 (size=12001) 2024-11-16T10:15:49,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/aec028b42e1b47c89d2e480d686a7727 2024-11-16T10:15:49,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/f8d2e7a0906a461fb1ce3f9705218436 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/f8d2e7a0906a461fb1ce3f9705218436 2024-11-16T10:15:49,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/f8d2e7a0906a461fb1ce3f9705218436, entries=350, sequenceid=76, filesize=63.8 K 2024-11-16T10:15:49,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/12a8133be48841a4bb5591c4eac582b1 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/12a8133be48841a4bb5591c4eac582b1 2024-11-16T10:15:49,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/12a8133be48841a4bb5591c4eac582b1, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T10:15:49,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/aec028b42e1b47c89d2e480d686a7727 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/aec028b42e1b47c89d2e480d686a7727 2024-11-16T10:15:49,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/aec028b42e1b47c89d2e480d686a7727, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T10:15:49,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~355.25 KB/363776, currentSize=53.67 KB/54960 for 6534162d1894b79648a1282b0163350b in 1372ms, sequenceid=76, compaction requested=true 2024-11-16T10:15:49,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:49,567 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:49,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:15:49,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:49,567 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:49,573 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:49,573 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/B is initiating minor compaction (all files) 2024-11-16T10:15:49,573 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/B in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:49,573 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/1c339bfbd4e84479b8509503c86b95cd, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/3ac1ff9c083d4491bc84eb057c1f2a03, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/12a8133be48841a4bb5591c4eac582b1] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=35.2 K 2024-11-16T10:15:49,574 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c339bfbd4e84479b8509503c86b95cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731752147364 2024-11-16T10:15:49,574 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ac1ff9c083d4491bc84eb057c1f2a03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731752148045 2024-11-16T10:15:49,574 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 12a8133be48841a4bb5591c4eac582b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731752148162 2024-11-16T10:15:49,576 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 144421 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:49,576 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/A is initiating minor compaction (all files) 2024-11-16T10:15:49,576 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/A in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:49,577 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/e829c5a1bd2b4bea9f422d6833ec6aa0, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/6c6d61ccc9a8457fb56a467f4a9de079, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/f8d2e7a0906a461fb1ce3f9705218436] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=141.0 K 2024-11-16T10:15:49,577 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:49,577 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. files: [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/e829c5a1bd2b4bea9f422d6833ec6aa0, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/6c6d61ccc9a8457fb56a467f4a9de079, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/f8d2e7a0906a461fb1ce3f9705218436] 2024-11-16T10:15:49,579 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting e829c5a1bd2b4bea9f422d6833ec6aa0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731752147358 2024-11-16T10:15:49,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:15:49,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:49,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:15:49,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:49,580 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c6d61ccc9a8457fb56a467f4a9de079, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731752148043 2024-11-16T10:15:49,584 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8d2e7a0906a461fb1ce3f9705218436, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731752148142 2024-11-16T10:15:49,597 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#B#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:49,597 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/3fb0101739d14a54859011d134b7a4c8 is 50, key is test_row_0/B:col10/1731752148192/Put/seqid=0 2024-11-16T10:15:49,617 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:49,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742175_1351 (size=12104) 2024-11-16T10:15:49,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:15:49,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:49,649 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 6534162d1894b79648a1282b0163350b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T10:15:49,649 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116dcaa445f0cb4465e94552f54f91704d7_6534162d1894b79648a1282b0163350b store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:49,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=A 2024-11-16T10:15:49,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:49,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=B 2024-11-16T10:15:49,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:49,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=C 2024-11-16T10:15:49,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:49,651 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116dcaa445f0cb4465e94552f54f91704d7_6534162d1894b79648a1282b0163350b, store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:49,652 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116dcaa445f0cb4465e94552f54f91704d7_6534162d1894b79648a1282b0163350b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:49,657 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/3fb0101739d14a54859011d134b7a4c8 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/3fb0101739d14a54859011d134b7a4c8 2024-11-16T10:15:49,663 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6534162d1894b79648a1282b0163350b/B of 6534162d1894b79648a1282b0163350b into 3fb0101739d14a54859011d134b7a4c8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:49,663 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:49,663 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/B, priority=13, startTime=1731752149567; duration=0sec 2024-11-16T10:15:49,663 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:49,663 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:B 2024-11-16T10:15:49,663 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:49,665 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:49,665 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/C is initiating minor compaction (all files) 2024-11-16T10:15:49,665 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/C in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:49,665 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/12e5403801704924b5792f46c6714808, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/106301c006ce453e9b20a62341abb688, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/aec028b42e1b47c89d2e480d686a7727] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=35.2 K 2024-11-16T10:15:49,665 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 12e5403801704924b5792f46c6714808, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731752147364 2024-11-16T10:15:49,666 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 106301c006ce453e9b20a62341abb688, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731752148045 2024-11-16T10:15:49,672 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting aec028b42e1b47c89d2e480d686a7727, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731752148162 2024-11-16T10:15:49,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111668f5d750acdf4d3ab81c5f8d7c0416d4_6534162d1894b79648a1282b0163350b is 50, key is test_row_0/A:col10/1731752148247/Put/seqid=0 2024-11-16T10:15:49,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742176_1352 (size=4469) 2024-11-16T10:15:49,691 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#C#compaction#306 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:49,691 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/b7d807a6c6f8445dbeb657556861d3a3 is 50, key is test_row_0/C:col10/1731752148192/Put/seqid=0 2024-11-16T10:15:49,694 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#A#compaction#304 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:49,694 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/3ad50cbae15d41fe8265a7992fff9056 is 175, key is test_row_0/A:col10/1731752148192/Put/seqid=0 2024-11-16T10:15:49,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742177_1353 (size=12154) 2024-11-16T10:15:49,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742178_1354 (size=12104) 2024-11-16T10:15:49,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742179_1355 (size=31058) 2024-11-16T10:15:49,731 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/b7d807a6c6f8445dbeb657556861d3a3 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/b7d807a6c6f8445dbeb657556861d3a3 2024-11-16T10:15:49,736 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/3ad50cbae15d41fe8265a7992fff9056 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/3ad50cbae15d41fe8265a7992fff9056 2024-11-16T10:15:49,739 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6534162d1894b79648a1282b0163350b/C of 6534162d1894b79648a1282b0163350b into b7d807a6c6f8445dbeb657556861d3a3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:49,740 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:49,740 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/C, priority=13, startTime=1731752149579; duration=0sec 2024-11-16T10:15:49,740 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:49,740 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:C 2024-11-16T10:15:49,744 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6534162d1894b79648a1282b0163350b/A of 6534162d1894b79648a1282b0163350b into 3ad50cbae15d41fe8265a7992fff9056(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:49,744 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:49,744 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/A, priority=13, startTime=1731752149567; duration=0sec 2024-11-16T10:15:49,744 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:49,744 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:A 2024-11-16T10:15:50,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:50,101 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111668f5d750acdf4d3ab81c5f8d7c0416d4_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111668f5d750acdf4d3ab81c5f8d7c0416d4_6534162d1894b79648a1282b0163350b 2024-11-16T10:15:50,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/f6490f78334d4fb38b25b75647850b55, store: [table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:50,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/f6490f78334d4fb38b25b75647850b55 is 175, key is test_row_0/A:col10/1731752148247/Put/seqid=0 2024-11-16T10:15:50,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742180_1356 (size=30955) 2024-11-16T10:15:50,110 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=87, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/f6490f78334d4fb38b25b75647850b55 2024-11-16T10:15:50,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/ce21ac41730b499a94586b92ec74cad8 is 50, key is test_row_0/B:col10/1731752148247/Put/seqid=0 2024-11-16T10:15:50,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742181_1357 (size=12001) 2024-11-16T10:15:50,124 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/ce21ac41730b499a94586b92ec74cad8 2024-11-16T10:15:50,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/0a287c1205c543488d9c8ac0e739aa72 is 50, key is test_row_0/C:col10/1731752148247/Put/seqid=0 2024-11-16T10:15:50,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742182_1358 (size=12001) 2024-11-16T10:15:50,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:50,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:50,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:50,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:50,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752210464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:50,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752210463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:50,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:50,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:50,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1731752150453, max timestamp=1731752150475Num uniques 859; 2024-11-16T10:15:50,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9556, totalHeapSize=21976, min timestamp=1731752150453, max timestamp=1731752150475Num uniques 859; 2024-11-16T10:15:50,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1731752150453, max timestamp=1731752150475Num uniques 859; 2024-11-16T10:15:50,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39326, new segment data size=9556, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:15:50,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:15:50,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:15:50,539 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/0a287c1205c543488d9c8ac0e739aa72 2024-11-16T10:15:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/f6490f78334d4fb38b25b75647850b55 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/f6490f78334d4fb38b25b75647850b55 2024-11-16T10:15:50,545 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/f6490f78334d4fb38b25b75647850b55, entries=150, sequenceid=87, filesize=30.2 K 2024-11-16T10:15:50,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/ce21ac41730b499a94586b92ec74cad8 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/ce21ac41730b499a94586b92ec74cad8 2024-11-16T10:15:50,550 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/ce21ac41730b499a94586b92ec74cad8, entries=150, sequenceid=87, filesize=11.7 K 2024-11-16T10:15:50,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/0a287c1205c543488d9c8ac0e739aa72 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/0a287c1205c543488d9c8ac0e739aa72 2024-11-16T10:15:50,554 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/0a287c1205c543488d9c8ac0e739aa72, entries=150, sequenceid=87, filesize=11.7 K 2024-11-16T10:15:50,562 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=107.34 KB/109920 for 6534162d1894b79648a1282b0163350b in 914ms, sequenceid=87, compaction requested=false 2024-11-16T10:15:50,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:50,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:50,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-16T10:15:50,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:50,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6534162d1894b79648a1282b0163350b 3/3 column families, dataSize=114.05 KB heapSize=320.53 KB 2024-11-16T10:15:50,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-16T10:15:50,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=A 2024-11-16T10:15:50,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:50,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=B 2024-11-16T10:15:50,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:50,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=C 2024-11-16T10:15:50,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:50,566 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-16T10:15:50,566 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2130 sec 2024-11-16T10:15:50,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 3.2190 sec 2024-11-16T10:15:50,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167ce7d4ca7c184079abac0c8e504e3fe2_6534162d1894b79648a1282b0163350b is 50, key is test_row_0/A:col10/1731752150550/Put/seqid=0 2024-11-16T10:15:50,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742183_1359 (size=17034) 2024-11-16T10:15:50,573 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:50,577 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167ce7d4ca7c184079abac0c8e504e3fe2_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167ce7d4ca7c184079abac0c8e504e3fe2_6534162d1894b79648a1282b0163350b 2024-11-16T10:15:50,577 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/7b38cfb2cd8d40b4b61fad7834ef1370, store: [table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:50,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/7b38cfb2cd8d40b4b61fad7834ef1370 is 175, key is test_row_0/A:col10/1731752150550/Put/seqid=0 2024-11-16T10:15:50,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742184_1360 (size=48139) 2024-11-16T10:15:50,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:50,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752210599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:50,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:50,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752210600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:50,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:50,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:50,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752210604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:50,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:50,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:50,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752210604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:50,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:50,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752210607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:50,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752210723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:50,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752210724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:50,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752210724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:50,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752210824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:50,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752210824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:50,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752210934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:50,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:50,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752210934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:50,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:50,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:50,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752210934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:50,987 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=123, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/7b38cfb2cd8d40b4b61fad7834ef1370 2024-11-16T10:15:50,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/5ca45a6f22764bde8ea347110a71240d is 50, key is test_row_0/B:col10/1731752150550/Put/seqid=0 2024-11-16T10:15:51,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742185_1361 (size=12001) 2024-11-16T10:15:51,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752211143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:51,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:51,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:51,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752211144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:51,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:51,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:51,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:51,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:51,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752211253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:51,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:51,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752211254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:51,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:51,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:51,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:51,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:51,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:51,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752211254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:51,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:51,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:51,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:51,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/5ca45a6f22764bde8ea347110a71240d 2024-11-16T10:15:51,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/7ac61e6e2b664427a19a04f577224723 is 50, key is test_row_0/C:col10/1731752150550/Put/seqid=0 2024-11-16T10:15:51,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742186_1362 (size=12001) 2024-11-16T10:15:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T10:15:51,504 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:15:51,505 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:15:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-16T10:15:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T10:15:51,506 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:15:51,507 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:15:51,507 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:15:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T10:15:51,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T10:15:51,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:51,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:51,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:51,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:51,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:51,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752211664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:51,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752211664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:51,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752211774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:51,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752211774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:51,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752211774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:51,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T10:15:51,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:51,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:51,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:51,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:51,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:51,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/7ac61e6e2b664427a19a04f577224723 2024-11-16T10:15:51,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/7b38cfb2cd8d40b4b61fad7834ef1370 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/7b38cfb2cd8d40b4b61fad7834ef1370 2024-11-16T10:15:51,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/7b38cfb2cd8d40b4b61fad7834ef1370, entries=250, sequenceid=123, filesize=47.0 K 2024-11-16T10:15:51,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/5ca45a6f22764bde8ea347110a71240d as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/5ca45a6f22764bde8ea347110a71240d 2024-11-16T10:15:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T10:15:51,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/5ca45a6f22764bde8ea347110a71240d, entries=150, sequenceid=123, filesize=11.7 K 2024-11-16T10:15:51,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/7ac61e6e2b664427a19a04f577224723 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/7ac61e6e2b664427a19a04f577224723 2024-11-16T10:15:51,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/7ac61e6e2b664427a19a04f577224723, entries=150, sequenceid=123, filesize=11.7 K 2024-11-16T10:15:51,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~320.48 KB/328176, currentSize=67.09 KB/68700 for 6534162d1894b79648a1282b0163350b in 1266ms, sequenceid=123, compaction requested=true 2024-11-16T10:15:51,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:51,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:15:51,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:51,830 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:51,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:15:51,830 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:51,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:51,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:15:51,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:51,832 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:51,832 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:51,832 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/B is initiating minor compaction (all files) 2024-11-16T10:15:51,832 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/A is initiating minor compaction (all files) 2024-11-16T10:15:51,832 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/B in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:51,832 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/A in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:51,832 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/3ad50cbae15d41fe8265a7992fff9056, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/f6490f78334d4fb38b25b75647850b55, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/7b38cfb2cd8d40b4b61fad7834ef1370] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=107.6 K 2024-11-16T10:15:51,832 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/3fb0101739d14a54859011d134b7a4c8, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/ce21ac41730b499a94586b92ec74cad8, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/5ca45a6f22764bde8ea347110a71240d] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=35.3 K 2024-11-16T10:15:51,832 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:51,832 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. files: [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/3ad50cbae15d41fe8265a7992fff9056, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/f6490f78334d4fb38b25b75647850b55, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/7b38cfb2cd8d40b4b61fad7834ef1370] 2024-11-16T10:15:51,832 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ad50cbae15d41fe8265a7992fff9056, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731752148162 2024-11-16T10:15:51,832 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fb0101739d14a54859011d134b7a4c8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731752148162 2024-11-16T10:15:51,833 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting ce21ac41730b499a94586b92ec74cad8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731752148207 2024-11-16T10:15:51,833 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6490f78334d4fb38b25b75647850b55, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731752148207 2024-11-16T10:15:51,833 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ca45a6f22764bde8ea347110a71240d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731752150547 2024-11-16T10:15:51,833 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b38cfb2cd8d40b4b61fad7834ef1370, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731752150518 2024-11-16T10:15:51,839 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#B#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:51,839 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/fc0779b78b0c42ceab2d594e8f69950b is 50, key is test_row_0/B:col10/1731752150550/Put/seqid=0 2024-11-16T10:15:51,840 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:51,843 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116498606fe39c3422d988160a949a53e65_6534162d1894b79648a1282b0163350b store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:51,844 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116498606fe39c3422d988160a949a53e65_6534162d1894b79648a1282b0163350b, store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:51,845 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116498606fe39c3422d988160a949a53e65_6534162d1894b79648a1282b0163350b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:51,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742187_1363 (size=12207) 2024-11-16T10:15:51,871 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/fc0779b78b0c42ceab2d594e8f69950b as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/fc0779b78b0c42ceab2d594e8f69950b 2024-11-16T10:15:51,875 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6534162d1894b79648a1282b0163350b/B of 6534162d1894b79648a1282b0163350b into fc0779b78b0c42ceab2d594e8f69950b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:51,875 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:51,875 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/B, priority=13, startTime=1731752151830; duration=0sec 2024-11-16T10:15:51,875 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:51,875 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:B 2024-11-16T10:15:51,875 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:51,877 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:51,877 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/C is initiating minor compaction (all files) 2024-11-16T10:15:51,877 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/C in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:51,878 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/b7d807a6c6f8445dbeb657556861d3a3, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/0a287c1205c543488d9c8ac0e739aa72, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/7ac61e6e2b664427a19a04f577224723] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=35.3 K 2024-11-16T10:15:51,878 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting b7d807a6c6f8445dbeb657556861d3a3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731752148162 2024-11-16T10:15:51,878 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a287c1205c543488d9c8ac0e739aa72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731752148207 2024-11-16T10:15:51,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742188_1364 (size=4469) 2024-11-16T10:15:51,879 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ac61e6e2b664427a19a04f577224723, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731752150547 2024-11-16T10:15:51,880 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#A#compaction#313 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:51,880 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/da4597c79f1e407389cae00d601dd268 is 175, key is test_row_0/A:col10/1731752150550/Put/seqid=0 2024-11-16T10:15:51,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742189_1365 (size=31161) 2024-11-16T10:15:51,918 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#C#compaction#314 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:51,919 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/a14597648fc7444a879bf7d078556a46 is 50, key is test_row_0/C:col10/1731752150550/Put/seqid=0 2024-11-16T10:15:51,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742190_1366 (size=12207) 2024-11-16T10:15:51,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T10:15:51,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:51,966 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 6534162d1894b79648a1282b0163350b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T10:15:51,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=A 2024-11-16T10:15:51,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:51,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=B 2024-11-16T10:15:51,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:51,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=C 2024-11-16T10:15:51,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:51,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116152a7168b402421f81871c5bbae70a1e_6534162d1894b79648a1282b0163350b is 50, key is test_row_0/A:col10/1731752150577/Put/seqid=0 2024-11-16T10:15:52,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742191_1367 (size=12254) 2024-11-16T10:15:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T10:15:52,324 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/da4597c79f1e407389cae00d601dd268 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/da4597c79f1e407389cae00d601dd268 2024-11-16T10:15:52,329 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6534162d1894b79648a1282b0163350b/A of 6534162d1894b79648a1282b0163350b into da4597c79f1e407389cae00d601dd268(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:52,329 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:52,329 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/A, priority=13, startTime=1731752151830; duration=0sec 2024-11-16T10:15:52,329 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:52,329 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:A 2024-11-16T10:15:52,345 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/a14597648fc7444a879bf7d078556a46 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/a14597648fc7444a879bf7d078556a46 2024-11-16T10:15:52,350 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6534162d1894b79648a1282b0163350b/C of 6534162d1894b79648a1282b0163350b into a14597648fc7444a879bf7d078556a46(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:52,350 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:52,350 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/C, priority=13, startTime=1731752151830; duration=0sec 2024-11-16T10:15:52,350 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:52,350 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:C 2024-11-16T10:15:52,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,407 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116152a7168b402421f81871c5bbae70a1e_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116152a7168b402421f81871c5bbae70a1e_6534162d1894b79648a1282b0163350b 2024-11-16T10:15:52,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/4ad1e48a969c40268ccea858ac4523d0, store: [table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:52,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/4ad1e48a969c40268ccea858ac4523d0 is 175, key is test_row_0/A:col10/1731752150577/Put/seqid=0 2024-11-16T10:15:52,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742192_1368 (size=31055) 2024-11-16T10:15:52,433 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/4ad1e48a969c40268ccea858ac4523d0 2024-11-16T10:15:52,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/b6bd75e09ba4483a8edf7ce274dd4e3d is 50, key is test_row_0/B:col10/1731752150577/Put/seqid=0 2024-11-16T10:15:52,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742193_1369 (size=12101) 2024-11-16T10:15:52,459 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/b6bd75e09ba4483a8edf7ce274dd4e3d 2024-11-16T10:15:52,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/b3b18aadd01f480687211cf8085d5999 is 50, key is test_row_0/C:col10/1731752150577/Put/seqid=0 2024-11-16T10:15:52,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742194_1370 (size=12101) 2024-11-16T10:15:52,481 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/b3b18aadd01f480687211cf8085d5999 2024-11-16T10:15:52,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/4ad1e48a969c40268ccea858ac4523d0 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/4ad1e48a969c40268ccea858ac4523d0 2024-11-16T10:15:52,492 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/4ad1e48a969c40268ccea858ac4523d0, entries=150, sequenceid=137, filesize=30.3 K 2024-11-16T10:15:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/b6bd75e09ba4483a8edf7ce274dd4e3d as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/b6bd75e09ba4483a8edf7ce274dd4e3d 2024-11-16T10:15:52,497 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/b6bd75e09ba4483a8edf7ce274dd4e3d, entries=150, sequenceid=137, filesize=11.8 K 2024-11-16T10:15:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/b3b18aadd01f480687211cf8085d5999 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/b3b18aadd01f480687211cf8085d5999 2024-11-16T10:15:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,502 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/b3b18aadd01f480687211cf8085d5999, entries=150, sequenceid=137, filesize=11.8 K 2024-11-16T10:15:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,502 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 6534162d1894b79648a1282b0163350b in 536ms, sequenceid=137, compaction requested=false 2024-11-16T10:15:52,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:52,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:52,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-16T10:15:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-16T10:15:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-16T10:15:52,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 997 msec 2024-11-16T10:15:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,507 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.0010 sec 2024-11-16T10:15:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T10:15:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,644 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:15:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,645 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:15:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-16T10:15:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,646 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:15:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T10:15:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,647 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:15:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,647 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:15:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T10:15:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T10:15:52,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:52,801 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 6534162d1894b79648a1282b0163350b 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-16T10:15:52,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=A 2024-11-16T10:15:52,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:52,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=B 2024-11-16T10:15:52,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:52,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=C 2024-11-16T10:15:52,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c6686b491d6c45e0b739db43d2f407d4_6534162d1894b79648a1282b0163350b is 50, key is test_row_2/A:col10/1731752152770/Put/seqid=0 2024-11-16T10:15:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742195_1371 (size=9814) 2024-11-16T10:15:52,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,856 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c6686b491d6c45e0b739db43d2f407d4_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c6686b491d6c45e0b739db43d2f407d4_6534162d1894b79648a1282b0163350b 2024-11-16T10:15:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/8d4e60376b8f4717a1101210e722da20, store: [table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/8d4e60376b8f4717a1101210e722da20 is 175, key is test_row_2/A:col10/1731752152770/Put/seqid=0 2024-11-16T10:15:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742196_1372 (size=22459) 2024-11-16T10:15:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:52,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:52,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731752152922, max timestamp=1731752152940Num uniques 859; 2024-11-16T10:15:52,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731752152922, max timestamp=1731752152940Num uniques 859; 2024-11-16T10:15:52,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T10:15:52,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T10:15:52,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731752152922, max timestamp=1731752152940Num uniques 859; 2024-11-16T10:15:52,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T10:15:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T10:15:53,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1731752152961, max timestamp=1731752153000Num uniques 1118; 2024-11-16T10:15:53,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=16848, suffix heap size=144240,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=368 2024-11-16T10:15:53,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731752152981, max timestamp=1731752153000Num uniques 1118; 2024-11-16T10:15:53,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-16T10:15:53,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731752152981, max timestamp=1731752153000Num uniques 1118; 2024-11-16T10:15:53,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-16T10:15:53,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731752153027, max timestamp=1731752153049Num uniques 1227; 2024-11-16T10:15:53,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=56192, new segment data size=12682, suffix heap size=155576,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1227, new segment cells count=277 2024-11-16T10:15:53,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731752153027, max timestamp=1731752153049Num uniques 1127; 2024-11-16T10:15:53,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-11-16T10:15:53,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731752153027, max timestamp=1731752153049Num uniques 1127; 2024-11-16T10:15:53,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-11-16T10:15:53,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731752153073, max timestamp=1731752153104Num uniques 1136; 2024-11-16T10:15:53,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-11-16T10:15:53,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731752153073, max timestamp=1731752153104Num uniques 1136; 2024-11-16T10:15:53,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-11-16T10:15:53,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731752153073, max timestamp=1731752153104Num uniques 1136; 2024-11-16T10:15:53,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-11-16T10:15:53,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731752153122, max timestamp=1731752153136Num uniques 1145; 2024-11-16T10:15:53,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-11-16T10:15:53,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731752153122, max timestamp=1731752153136Num uniques 1145; 2024-11-16T10:15:53,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-11-16T10:15:53,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731752153122, max timestamp=1731752153136Num uniques 1145; 2024-11-16T10:15:53,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-11-16T10:15:53,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:53,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752213167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:53,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731752153164, max timestamp=1731752153177Num uniques 1154; 2024-11-16T10:15:53,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:53,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=11630, suffix heap size=147984,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=254 2024-11-16T10:15:53,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731752153164, max timestamp=1731752153177Num uniques 1154; 2024-11-16T10:15:53,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=11630, suffix heap size=147984,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=254 2024-11-16T10:15:53,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731752153164, max timestamp=1731752153177Num uniques 1154; 2024-11-16T10:15:53,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=11630, suffix heap size=147984,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=254 2024-11-16T10:15:53,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1731752153206, max timestamp=1731752153219Num uniques 1113; 2024-11-16T10:15:53,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1731752153206, max timestamp=1731752153219Num uniques 1113; 2024-11-16T10:15:53,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1731752153206, max timestamp=1731752153219Num uniques 1113; 2024-11-16T10:15:53,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-11-16T10:15:53,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-11-16T10:15:53,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-11-16T10:15:53,266 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=144, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/8d4e60376b8f4717a1101210e722da20 2024-11-16T10:15:53,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/b2fbcf4acb564ffc90b0986b2e6a4e37 is 50, key is test_row_2/B:col10/1731752152770/Put/seqid=0 2024-11-16T10:15:53,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731752153258, max timestamp=1731752153271Num uniques 1122; 2024-11-16T10:15:53,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-11-16T10:15:53,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731752153258, max timestamp=1731752153272Num uniques 1122; 2024-11-16T10:15:53,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-11-16T10:15:53,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731752153258, max timestamp=1731752153272Num uniques 1122; 2024-11-16T10:15:53,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-11-16T10:15:53,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T10:15:53,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742197_1373 (size=7365) 2024-11-16T10:15:53,279 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/b2fbcf4acb564ffc90b0986b2e6a4e37 2024-11-16T10:15:53,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/37fa3907eefb4d2a8e26a8970c38f0c0 is 50, key is test_row_2/C:col10/1731752152770/Put/seqid=0 2024-11-16T10:15:53,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742198_1374 (size=7365) 2024-11-16T10:15:53,294 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/37fa3907eefb4d2a8e26a8970c38f0c0 2024-11-16T10:15:53,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/8d4e60376b8f4717a1101210e722da20 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/8d4e60376b8f4717a1101210e722da20 2024-11-16T10:15:53,301 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/8d4e60376b8f4717a1101210e722da20, entries=100, sequenceid=144, filesize=21.9 K 2024-11-16T10:15:53,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/b2fbcf4acb564ffc90b0986b2e6a4e37 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/b2fbcf4acb564ffc90b0986b2e6a4e37 2024-11-16T10:15:53,312 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/b2fbcf4acb564ffc90b0986b2e6a4e37, entries=50, sequenceid=144, filesize=7.2 K 2024-11-16T10:15:53,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/37fa3907eefb4d2a8e26a8970c38f0c0 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/37fa3907eefb4d2a8e26a8970c38f0c0 2024-11-16T10:15:53,322 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/37fa3907eefb4d2a8e26a8970c38f0c0, entries=50, sequenceid=144, filesize=7.2 K 2024-11-16T10:15:53,326 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=127.47 KB/130530 for 6534162d1894b79648a1282b0163350b in 526ms, sequenceid=144, compaction requested=true 2024-11-16T10:15:53,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:53,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:53,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-16T10:15:53,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:53,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6534162d1894b79648a1282b0163350b 3/3 column families, dataSize=134.18 KB heapSize=383.77 KB 2024-11-16T10:15:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-16T10:15:53,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=A 2024-11-16T10:15:53,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:53,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=B 2024-11-16T10:15:53,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:53,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=C 2024-11-16T10:15:53,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:53,336 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-16T10:15:53,336 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 681 msec 2024-11-16T10:15:53,337 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 691 msec 2024-11-16T10:15:53,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:53,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752213335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:53,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752213336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:53,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167497f3bb3b1945a0afbc226ef39fad5f_6534162d1894b79648a1282b0163350b is 50, key is test_row_0/A:col10/1731752153316/Put/seqid=0 2024-11-16T10:15:53,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:53,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752213342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:53,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:53,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752213342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:53,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752213343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:53,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:53,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:53,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742199_1375 (size=17534) 2024-11-16T10:15:53,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:53,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752213454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:53,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:53,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752213454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:53,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752213454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:53,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752213454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:53,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:53,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752213464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:53,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:53,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:53,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752213677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:53,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:53,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752213677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:53,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752213677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:53,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752213677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:53,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:53,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752213684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:53,758 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:53,762 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167497f3bb3b1945a0afbc226ef39fad5f_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167497f3bb3b1945a0afbc226ef39fad5f_6534162d1894b79648a1282b0163350b 2024-11-16T10:15:53,763 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/66f6b2c9540c421da8f2c9947c1aca19, store: [table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:53,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/66f6b2c9540c421da8f2c9947c1aca19 is 175, key is test_row_0/A:col10/1731752153316/Put/seqid=0 2024-11-16T10:15:53,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742200_1376 (size=48639) 2024-11-16T10:15:53,773 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=301, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/66f6b2c9540c421da8f2c9947c1aca19 2024-11-16T10:15:53,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/6b6f5bd33f2e421286f10ea0a1bbe96b is 50, key is test_row_0/B:col10/1731752153316/Put/seqid=0 2024-11-16T10:15:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T10:15:53,785 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:15:53,786 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:15:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-16T10:15:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T10:15:53,787 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:15:53,788 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:15:53,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:15:53,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742201_1377 (size=12301) 2024-11-16T10:15:53,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T10:15:53,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T10:15:53,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:53,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:53,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:53,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:53,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:53,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:53,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752213994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:53,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:53,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752213994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:53,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:53,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752213994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:53,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:53,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:54,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:54,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752214004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:54,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:54,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:54,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:54,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:54,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752214004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:54,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:54,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:54,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:54,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T10:15:54,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:54,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:54,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:54,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:54,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:54,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T10:15:54,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/6b6f5bd33f2e421286f10ea0a1bbe96b 2024-11-16T10:15:54,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/d2f0fb80ecea405e9ebbca846cab8653 is 50, key is test_row_0/C:col10/1731752153316/Put/seqid=0 2024-11-16T10:15:54,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742202_1378 (size=12301) 2024-11-16T10:15:54,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T10:15:54,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:54,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:54,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:54,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:54,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:54,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T10:15:54,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:54,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:54,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:54,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:54,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:54,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T10:15:54,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752214504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:54,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:54,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:54,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752214505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:54,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:54,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752214505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:54,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752214514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:54,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:54,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:54,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:54,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752214524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:54,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:54,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:54,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:54,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T10:15:54,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:54,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:54,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:54,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:54,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:54,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/d2f0fb80ecea405e9ebbca846cab8653 2024-11-16T10:15:54,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/66f6b2c9540c421da8f2c9947c1aca19 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/66f6b2c9540c421da8f2c9947c1aca19 2024-11-16T10:15:54,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/66f6b2c9540c421da8f2c9947c1aca19, entries=250, sequenceid=301, filesize=47.5 K 2024-11-16T10:15:54,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/6b6f5bd33f2e421286f10ea0a1bbe96b as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/6b6f5bd33f2e421286f10ea0a1bbe96b 2024-11-16T10:15:54,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/6b6f5bd33f2e421286f10ea0a1bbe96b, entries=150, sequenceid=301, filesize=12.0 K 2024-11-16T10:15:54,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/d2f0fb80ecea405e9ebbca846cab8653 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/d2f0fb80ecea405e9ebbca846cab8653 2024-11-16T10:15:54,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/d2f0fb80ecea405e9ebbca846cab8653, entries=150, sequenceid=301, filesize=12.0 K 2024-11-16T10:15:54,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~383.72 KB/392928, currentSize=46.96 KB/48090 for 6534162d1894b79648a1282b0163350b in 1296ms, sequenceid=301, compaction requested=true 2024-11-16T10:15:54,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:54,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:15:54,623 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:15:54,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:54,623 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:15:54,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:15:54,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:54,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:15:54,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:54,626 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43974 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:15:54,626 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/B is initiating minor compaction (all files) 2024-11-16T10:15:54,626 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/B in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:54,626 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/fc0779b78b0c42ceab2d594e8f69950b, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/b6bd75e09ba4483a8edf7ce274dd4e3d, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/b2fbcf4acb564ffc90b0986b2e6a4e37, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/6b6f5bd33f2e421286f10ea0a1bbe96b] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=42.9 K 2024-11-16T10:15:54,626 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133314 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:15:54,626 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/A is initiating minor compaction (all files) 2024-11-16T10:15:54,626 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/A in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:54,626 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting fc0779b78b0c42ceab2d594e8f69950b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731752150547 2024-11-16T10:15:54,626 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/da4597c79f1e407389cae00d601dd268, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/4ad1e48a969c40268ccea858ac4523d0, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/8d4e60376b8f4717a1101210e722da20, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/66f6b2c9540c421da8f2c9947c1aca19] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=130.2 K 2024-11-16T10:15:54,626 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:54,626 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. files: [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/da4597c79f1e407389cae00d601dd268, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/4ad1e48a969c40268ccea858ac4523d0, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/8d4e60376b8f4717a1101210e722da20, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/66f6b2c9540c421da8f2c9947c1aca19] 2024-11-16T10:15:54,627 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting b6bd75e09ba4483a8edf7ce274dd4e3d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731752150577 2024-11-16T10:15:54,627 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting da4597c79f1e407389cae00d601dd268, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731752150547 2024-11-16T10:15:54,627 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting b2fbcf4acb564ffc90b0986b2e6a4e37, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731752152770 2024-11-16T10:15:54,627 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ad1e48a969c40268ccea858ac4523d0, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731752150577 2024-11-16T10:15:54,627 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b6f5bd33f2e421286f10ea0a1bbe96b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731752153316 2024-11-16T10:15:54,627 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d4e60376b8f4717a1101210e722da20, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731752152769 2024-11-16T10:15:54,628 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66f6b2c9540c421da8f2c9947c1aca19, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731752153292 2024-11-16T10:15:54,634 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:54,635 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#B#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:54,636 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/36d749fc3c714b51871f13da11cc5603 is 50, key is test_row_0/B:col10/1731752153316/Put/seqid=0 2024-11-16T10:15:54,636 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111674eddeba65dc490882e4703bf94cc46d_6534162d1894b79648a1282b0163350b store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:54,639 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111674eddeba65dc490882e4703bf94cc46d_6534162d1894b79648a1282b0163350b, store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:54,639 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111674eddeba65dc490882e4703bf94cc46d_6534162d1894b79648a1282b0163350b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:54,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742204_1380 (size=4469) 2024-11-16T10:15:54,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742203_1379 (size=12643) 2024-11-16T10:15:54,645 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#A#compaction#325 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:54,646 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/4fe34563e7ee4cac9a50e9821105fbb5 is 175, key is test_row_0/A:col10/1731752153316/Put/seqid=0 2024-11-16T10:15:54,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742205_1381 (size=31597) 2024-11-16T10:15:54,656 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/36d749fc3c714b51871f13da11cc5603 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/36d749fc3c714b51871f13da11cc5603 2024-11-16T10:15:54,658 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/4fe34563e7ee4cac9a50e9821105fbb5 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/4fe34563e7ee4cac9a50e9821105fbb5 2024-11-16T10:15:54,661 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6534162d1894b79648a1282b0163350b/B of 6534162d1894b79648a1282b0163350b into 36d749fc3c714b51871f13da11cc5603(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:54,661 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:54,661 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/B, priority=12, startTime=1731752154623; duration=0sec 2024-11-16T10:15:54,661 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:54,661 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:B 2024-11-16T10:15:54,661 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:15:54,663 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43974 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:15:54,663 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6534162d1894b79648a1282b0163350b/A of 6534162d1894b79648a1282b0163350b into 4fe34563e7ee4cac9a50e9821105fbb5(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:54,663 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:54,663 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/C is initiating minor compaction (all files) 2024-11-16T10:15:54,663 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/A, priority=12, startTime=1731752154622; duration=0sec 2024-11-16T10:15:54,663 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/C in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:54,663 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:54,663 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:A 2024-11-16T10:15:54,663 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/a14597648fc7444a879bf7d078556a46, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/b3b18aadd01f480687211cf8085d5999, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/37fa3907eefb4d2a8e26a8970c38f0c0, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/d2f0fb80ecea405e9ebbca846cab8653] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=42.9 K 2024-11-16T10:15:54,663 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting a14597648fc7444a879bf7d078556a46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731752150547 2024-11-16T10:15:54,664 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting b3b18aadd01f480687211cf8085d5999, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731752150577 2024-11-16T10:15:54,664 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 37fa3907eefb4d2a8e26a8970c38f0c0, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731752152770 2024-11-16T10:15:54,664 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting d2f0fb80ecea405e9ebbca846cab8653, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731752153316 2024-11-16T10:15:54,671 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#C#compaction#326 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:54,671 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/15fee025f79847dc8731d3272035e745 is 50, key is test_row_0/C:col10/1731752153316/Put/seqid=0 2024-11-16T10:15:54,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742206_1382 (size=12643) 2024-11-16T10:15:54,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T10:15:54,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:54,705 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 6534162d1894b79648a1282b0163350b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T10:15:54,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=A 2024-11-16T10:15:54,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:54,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=B 2024-11-16T10:15:54,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:54,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=C 2024-11-16T10:15:54,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:54,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c910071cc1a044e48038aee25a68a63b_6534162d1894b79648a1282b0163350b is 50, key is test_row_0/A:col10/1731752153327/Put/seqid=0 2024-11-16T10:15:54,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742207_1383 (size=12454) 2024-11-16T10:15:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T10:15:55,081 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/15fee025f79847dc8731d3272035e745 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/15fee025f79847dc8731d3272035e745 2024-11-16T10:15:55,086 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6534162d1894b79648a1282b0163350b/C of 6534162d1894b79648a1282b0163350b into 15fee025f79847dc8731d3272035e745(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:55,086 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:55,086 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/C, priority=12, startTime=1731752154623; duration=0sec 2024-11-16T10:15:55,086 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:55,086 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:C 2024-11-16T10:15:55,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:55,123 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c910071cc1a044e48038aee25a68a63b_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c910071cc1a044e48038aee25a68a63b_6534162d1894b79648a1282b0163350b 2024-11-16T10:15:55,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/1d43aa79fbc5455bbc4bc25a9979cccb, store: [table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:55,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/1d43aa79fbc5455bbc4bc25a9979cccb is 175, key is test_row_0/A:col10/1731752153327/Put/seqid=0 2024-11-16T10:15:55,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742208_1384 (size=31255) 2024-11-16T10:15:55,136 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/1d43aa79fbc5455bbc4bc25a9979cccb 2024-11-16T10:15:55,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/9ec865232f5640c69d9e0c6de574ecec is 50, key is test_row_0/B:col10/1731752153327/Put/seqid=0 2024-11-16T10:15:55,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742209_1385 (size=12301) 2024-11-16T10:15:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:55,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:55,567 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/9ec865232f5640c69d9e0c6de574ecec 2024-11-16T10:15:55,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752155555, max timestamp=1731752155567Num uniques 859; 2024-11-16T10:15:55,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752155555, max timestamp=1731752155567Num uniques 859; 2024-11-16T10:15:55,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:15:55,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:15:55,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752155555, max timestamp=1731752155567Num uniques 859; 2024-11-16T10:15:55,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:15:55,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/fe50b08106424fb58e89b20a1cfc0d37 is 50, key is test_row_0/C:col10/1731752153327/Put/seqid=0 2024-11-16T10:15:55,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742210_1386 (size=12301) 2024-11-16T10:15:55,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752215627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:55,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752215637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:55,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752215639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:55,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:55,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752215640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:55,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:55,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752215640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:55,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752215743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:55,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752215744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:55,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752215754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:55,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752215754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:55,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:55,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752215764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:55,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T10:15:55,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752215954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752215954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:55,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752215964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:55,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:55,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752215964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:55,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:55,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:55,984 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/fe50b08106424fb58e89b20a1cfc0d37 2024-11-16T10:15:55,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/1d43aa79fbc5455bbc4bc25a9979cccb as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/1d43aa79fbc5455bbc4bc25a9979cccb 2024-11-16T10:15:55,990 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/1d43aa79fbc5455bbc4bc25a9979cccb, entries=150, sequenceid=313, filesize=30.5 K 2024-11-16T10:15:55,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/9ec865232f5640c69d9e0c6de574ecec as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/9ec865232f5640c69d9e0c6de574ecec 2024-11-16T10:15:55,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752215984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:55,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:55,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:55,995 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/9ec865232f5640c69d9e0c6de574ecec, entries=150, sequenceid=313, filesize=12.0 K 2024-11-16T10:15:55,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/fe50b08106424fb58e89b20a1cfc0d37 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/fe50b08106424fb58e89b20a1cfc0d37 2024-11-16T10:15:56,000 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/fe50b08106424fb58e89b20a1cfc0d37, entries=150, sequenceid=313, filesize=12.0 K 2024-11-16T10:15:56,001 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6534162d1894b79648a1282b0163350b in 1295ms, sequenceid=313, compaction requested=false 2024-11-16T10:15:56,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:56,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:56,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-16T10:15:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-16T10:15:56,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-16T10:15:56,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2170 sec 2024-11-16T10:15:56,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.2220 sec 2024-11-16T10:15:56,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:56,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6534162d1894b79648a1282b0163350b 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-16T10:15:56,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=A 2024-11-16T10:15:56,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:56,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=B 2024-11-16T10:15:56,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:56,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=C 2024-11-16T10:15:56,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:56,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e449814f90914fad8000b34ebe08fbb8_6534162d1894b79648a1282b0163350b is 50, key is test_row_0/A:col10/1731752155638/Put/seqid=0 2024-11-16T10:15:56,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752216290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:56,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742211_1387 (size=14994) 2024-11-16T10:15:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:56,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752216296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:56,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752216297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:56,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:56,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752216300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:56,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752216305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:56,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:56,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752216414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:56,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:56,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:56,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752216414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:56,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:56,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752216424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:56,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:56,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752216434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:56,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:56,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752216634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:56,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752216636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:56,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:56,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752216645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:56,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752216655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:56,710 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:56,717 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e449814f90914fad8000b34ebe08fbb8_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e449814f90914fad8000b34ebe08fbb8_6534162d1894b79648a1282b0163350b 2024-11-16T10:15:56,718 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/5bc94f196ff34e0992ed00c8f3f9e375, store: [table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:56,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/5bc94f196ff34e0992ed00c8f3f9e375 is 175, key is test_row_0/A:col10/1731752155638/Put/seqid=0 2024-11-16T10:15:56,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742212_1388 (size=39949) 2024-11-16T10:15:56,734 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/5bc94f196ff34e0992ed00c8f3f9e375 2024-11-16T10:15:56,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/089a507ad5144e92830f86f0cc2fc276 is 50, key is test_row_0/B:col10/1731752155638/Put/seqid=0 2024-11-16T10:15:56,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742213_1389 (size=12301) 2024-11-16T10:15:56,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752216824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:56,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752216944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:56,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752216964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:56,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:56,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752216965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:57,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752216984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:57,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:57,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:57,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:57,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/089a507ad5144e92830f86f0cc2fc276 2024-11-16T10:15:57,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/d25ce834cf544473863af106fdfcb965 is 50, key is test_row_0/C:col10/1731752155638/Put/seqid=0 2024-11-16T10:15:57,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742214_1390 (size=12301) 2024-11-16T10:15:57,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/d25ce834cf544473863af106fdfcb965 2024-11-16T10:15:57,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/5bc94f196ff34e0992ed00c8f3f9e375 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/5bc94f196ff34e0992ed00c8f3f9e375 2024-11-16T10:15:57,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/5bc94f196ff34e0992ed00c8f3f9e375, entries=200, sequenceid=351, filesize=39.0 K 2024-11-16T10:15:57,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/089a507ad5144e92830f86f0cc2fc276 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/089a507ad5144e92830f86f0cc2fc276 2024-11-16T10:15:57,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/089a507ad5144e92830f86f0cc2fc276, entries=150, sequenceid=351, filesize=12.0 K 2024-11-16T10:15:57,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/d25ce834cf544473863af106fdfcb965 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/d25ce834cf544473863af106fdfcb965 2024-11-16T10:15:57,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/d25ce834cf544473863af106fdfcb965, entries=150, sequenceid=351, filesize=12.0 K 2024-11-16T10:15:57,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 6534162d1894b79648a1282b0163350b in 996ms, sequenceid=351, compaction requested=true 2024-11-16T10:15:57,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:57,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:15:57,274 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:57,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:57,274 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:57,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:15:57,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:57,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:15:57,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:57,275 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102801 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:57,275 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:57,275 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/A is initiating minor compaction (all files) 2024-11-16T10:15:57,275 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/B is initiating minor compaction (all files) 2024-11-16T10:15:57,275 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/A in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:57,275 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/B in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:57,275 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/4fe34563e7ee4cac9a50e9821105fbb5, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/1d43aa79fbc5455bbc4bc25a9979cccb, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/5bc94f196ff34e0992ed00c8f3f9e375] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=100.4 K 2024-11-16T10:15:57,275 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/36d749fc3c714b51871f13da11cc5603, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/9ec865232f5640c69d9e0c6de574ecec, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/089a507ad5144e92830f86f0cc2fc276] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=36.4 K 2024-11-16T10:15:57,275 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:57,275 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. files: [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/4fe34563e7ee4cac9a50e9821105fbb5, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/1d43aa79fbc5455bbc4bc25a9979cccb, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/5bc94f196ff34e0992ed00c8f3f9e375] 2024-11-16T10:15:57,276 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fe34563e7ee4cac9a50e9821105fbb5, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731752153316 2024-11-16T10:15:57,276 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 36d749fc3c714b51871f13da11cc5603, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731752153316 2024-11-16T10:15:57,276 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d43aa79fbc5455bbc4bc25a9979cccb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731752153327 2024-11-16T10:15:57,276 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ec865232f5640c69d9e0c6de574ecec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731752153327 2024-11-16T10:15:57,276 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 089a507ad5144e92830f86f0cc2fc276, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731752155638 2024-11-16T10:15:57,276 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bc94f196ff34e0992ed00c8f3f9e375, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731752155637 2024-11-16T10:15:57,282 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:57,283 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#B#compaction#333 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:57,283 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116461d165b0f4b47ba9b40da6e1b93ada8_6534162d1894b79648a1282b0163350b store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:57,283 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/234830f5b2ac4c5f9a5aa5c19bfc2ae7 is 50, key is test_row_0/B:col10/1731752155638/Put/seqid=0 2024-11-16T10:15:57,285 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116461d165b0f4b47ba9b40da6e1b93ada8_6534162d1894b79648a1282b0163350b, store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:57,285 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116461d165b0f4b47ba9b40da6e1b93ada8_6534162d1894b79648a1282b0163350b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:57,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742215_1391 (size=12745) 2024-11-16T10:15:57,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742216_1392 (size=4469) 2024-11-16T10:15:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:57,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6534162d1894b79648a1282b0163350b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T10:15:57,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=A 2024-11-16T10:15:57,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:57,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=B 2024-11-16T10:15:57,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:57,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=C 2024-11-16T10:15:57,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:57,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111670cf7898f5cc4da289ba55d0f6edacfd_6534162d1894b79648a1282b0163350b is 50, key is test_row_0/A:col10/1731752157486/Put/seqid=0 2024-11-16T10:15:57,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742217_1393 (size=14994) 2024-11-16T10:15:57,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731752157559, max timestamp=1731752157612Num uniques 859; 2024-11-16T10:15:57,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T10:15:57,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731752157559, max timestamp=1731752157612Num uniques 859; 2024-11-16T10:15:57,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T10:15:57,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731752157559, max timestamp=1731752157612Num uniques 859; 2024-11-16T10:15:57,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T10:15:57,698 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#A#compaction#334 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:57,698 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/fac80d73c20341de87c95f7b14f06df5 is 175, key is test_row_0/A:col10/1731752155638/Put/seqid=0 2024-11-16T10:15:57,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:57,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752217667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:57,705 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/234830f5b2ac4c5f9a5aa5c19bfc2ae7 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/234830f5b2ac4c5f9a5aa5c19bfc2ae7 2024-11-16T10:15:57,711 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6534162d1894b79648a1282b0163350b/B of 6534162d1894b79648a1282b0163350b into 234830f5b2ac4c5f9a5aa5c19bfc2ae7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:57,711 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:57,711 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/B, priority=13, startTime=1731752157274; duration=0sec 2024-11-16T10:15:57,711 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:15:57,711 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:B 2024-11-16T10:15:57,712 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:15:57,712 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:15:57,713 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/C is initiating minor compaction (all files) 2024-11-16T10:15:57,713 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/C in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:57,713 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/15fee025f79847dc8731d3272035e745, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/fe50b08106424fb58e89b20a1cfc0d37, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/d25ce834cf544473863af106fdfcb965] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=36.4 K 2024-11-16T10:15:57,714 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 15fee025f79847dc8731d3272035e745, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731752153316 2024-11-16T10:15:57,714 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting fe50b08106424fb58e89b20a1cfc0d37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731752153327 2024-11-16T10:15:57,714 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting d25ce834cf544473863af106fdfcb965, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731752155638 2024-11-16T10:15:57,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742218_1394 (size=31699) 2024-11-16T10:15:57,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:57,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752217691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:57,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:57,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:57,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:57,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:57,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752217699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:57,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752217701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:57,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:57,735 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/fac80d73c20341de87c95f7b14f06df5 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/fac80d73c20341de87c95f7b14f06df5 2024-11-16T10:15:57,736 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#C#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:15:57,737 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/57b51273ebed4ac385c78587be1965ff is 50, key is test_row_0/C:col10/1731752155638/Put/seqid=0 2024-11-16T10:15:57,743 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6534162d1894b79648a1282b0163350b/A of 6534162d1894b79648a1282b0163350b into fac80d73c20341de87c95f7b14f06df5(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:57,743 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:57,743 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/A, priority=13, startTime=1731752157274; duration=0sec 2024-11-16T10:15:57,743 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:57,743 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:A 2024-11-16T10:15:57,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742219_1395 (size=12745) 2024-11-16T10:15:57,769 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/57b51273ebed4ac385c78587be1965ff as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/57b51273ebed4ac385c78587be1965ff 2024-11-16T10:15:57,774 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6534162d1894b79648a1282b0163350b/C of 6534162d1894b79648a1282b0163350b into 57b51273ebed4ac385c78587be1965ff(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:15:57,774 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:57,774 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/C, priority=13, startTime=1731752157275; duration=0sec 2024-11-16T10:15:57,775 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:15:57,775 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:C 2024-11-16T10:15:57,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:57,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752217804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:57,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:57,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752217834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:57,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:57,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:57,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:57,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:57,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752217834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:57,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:57,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752217835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:57,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:57,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:57,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:57,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:57,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:57,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:57,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:57,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752217854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:57,875 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in 6534162d1894b79648a1282b0163350b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:15:57,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:57,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:57,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:57,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T10:15:57,946 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:15:57,947 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:15:57,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-16T10:15:57,949 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:15:57,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T10:15:57,949 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:15:57,949 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:15:57,978 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:57,986 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111670cf7898f5cc4da289ba55d0f6edacfd_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111670cf7898f5cc4da289ba55d0f6edacfd_6534162d1894b79648a1282b0163350b 2024-11-16T10:15:57,988 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/4190a7032a8a4dcd9975836dde31feb0, store: [table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:57,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/4190a7032a8a4dcd9975836dde31feb0 is 175, key is test_row_0/A:col10/1731752157486/Put/seqid=0 2024-11-16T10:15:57,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742220_1396 (size=39949) 2024-11-16T10:15:57,994 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=361, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/4190a7032a8a4dcd9975836dde31feb0 2024-11-16T10:15:58,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/7792c44bb0af4a6e8e7368ee38b8b117 is 50, key is test_row_0/B:col10/1731752157486/Put/seqid=0 2024-11-16T10:15:58,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:58,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752218034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:58,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742221_1397 (size=12301) 2024-11-16T10:15:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:58,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T10:15:58,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:58,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752218054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:58,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752218054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:58,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752218054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:58,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T10:15:58,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:58,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:58,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:58,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:58,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:58,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:58,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T10:15:58,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:58,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:58,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:58,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:58,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:58,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T10:15:58,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752218354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:58,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:58,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752218375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752218375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752218375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:58,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T10:15:58,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:58,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:58,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:58,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:58,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:58,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:15:58,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/7792c44bb0af4a6e8e7368ee38b8b117 2024-11-16T10:15:58,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/af69d20769d146c0bc3a3038833c5784 is 50, key is test_row_0/C:col10/1731752157486/Put/seqid=0 2024-11-16T10:15:58,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742222_1398 (size=12301) 2024-11-16T10:15:58,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/af69d20769d146c0bc3a3038833c5784 2024-11-16T10:15:58,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/4190a7032a8a4dcd9975836dde31feb0 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/4190a7032a8a4dcd9975836dde31feb0 2024-11-16T10:15:58,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/4190a7032a8a4dcd9975836dde31feb0, entries=200, sequenceid=361, filesize=39.0 K 2024-11-16T10:15:58,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/7792c44bb0af4a6e8e7368ee38b8b117 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/7792c44bb0af4a6e8e7368ee38b8b117 2024-11-16T10:15:58,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/7792c44bb0af4a6e8e7368ee38b8b117, entries=150, sequenceid=361, filesize=12.0 K 2024-11-16T10:15:58,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/af69d20769d146c0bc3a3038833c5784 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/af69d20769d146c0bc3a3038833c5784 2024-11-16T10:15:58,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/af69d20769d146c0bc3a3038833c5784, entries=150, sequenceid=361, filesize=12.0 K 2024-11-16T10:15:58,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 6534162d1894b79648a1282b0163350b in 1069ms, sequenceid=361, compaction requested=false 2024-11-16T10:15:58,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:58,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T10:15:58,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:58,563 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 6534162d1894b79648a1282b0163350b 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-16T10:15:58,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=A 2024-11-16T10:15:58,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:58,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=B 2024-11-16T10:15:58,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:58,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=C 2024-11-16T10:15:58,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:15:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T10:15:58,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c8f040857e8444b88fe1697f03529c7f_6534162d1894b79648a1282b0163350b is 50, key is test_row_0/A:col10/1731752157697/Put/seqid=0 2024-11-16T10:15:58,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742223_1399 (size=12454) 2024-11-16T10:15:58,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:58,645 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c8f040857e8444b88fe1697f03529c7f_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c8f040857e8444b88fe1697f03529c7f_6534162d1894b79648a1282b0163350b 2024-11-16T10:15:58,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/24823500d81e49b59a0dd51de84bb56a, store: [table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:58,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/24823500d81e49b59a0dd51de84bb56a is 175, key is test_row_0/A:col10/1731752157697/Put/seqid=0 2024-11-16T10:15:58,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742224_1400 (size=31255) 2024-11-16T10:15:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:58,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:15:58,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752218920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:58,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:58,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:58,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752218923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752218922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:58,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752218923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:59,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:59,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752219034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:59,057 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=399, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/24823500d81e49b59a0dd51de84bb56a 2024-11-16T10:15:59,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/5dc7378cc088424fa493ac369f935156 is 50, key is test_row_0/B:col10/1731752157697/Put/seqid=0 2024-11-16T10:15:59,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:59,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752219055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:59,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:59,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752219055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:59,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:59,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:59,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:59,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752219055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:59,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:59,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742225_1401 (size=12301) 2024-11-16T10:15:59,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T10:15:59,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752219244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:59,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:59,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:59,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752219285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:59,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:59,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752219285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:59,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:59,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:59,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752219285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:59,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:59,476 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/5dc7378cc088424fa493ac369f935156 2024-11-16T10:15:59,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/d60d01f8a3994950ad0de5ab74ef2c83 is 50, key is test_row_0/C:col10/1731752157697/Put/seqid=0 2024-11-16T10:15:59,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742226_1402 (size=12301) 2024-11-16T10:15:59,498 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/d60d01f8a3994950ad0de5ab74ef2c83 2024-11-16T10:15:59,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/24823500d81e49b59a0dd51de84bb56a as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/24823500d81e49b59a0dd51de84bb56a 2024-11-16T10:15:59,508 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/24823500d81e49b59a0dd51de84bb56a, entries=150, sequenceid=399, filesize=30.5 K 2024-11-16T10:15:59,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/5dc7378cc088424fa493ac369f935156 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/5dc7378cc088424fa493ac369f935156 2024-11-16T10:15:59,512 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/5dc7378cc088424fa493ac369f935156, entries=150, sequenceid=399, filesize=12.0 K 2024-11-16T10:15:59,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/d60d01f8a3994950ad0de5ab74ef2c83 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/d60d01f8a3994950ad0de5ab74ef2c83 2024-11-16T10:15:59,516 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/d60d01f8a3994950ad0de5ab74ef2c83, entries=150, sequenceid=399, filesize=12.0 K 2024-11-16T10:15:59,516 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=53.67 KB/54960 for 6534162d1894b79648a1282b0163350b in 953ms, sequenceid=399, compaction requested=true 2024-11-16T10:15:59,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:15:59,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:15:59,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-16T10:15:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-16T10:15:59,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-16T10:15:59,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5700 sec 2024-11-16T10:15:59,524 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.5760 sec 2024-11-16T10:15:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 6534162d1894b79648a1282b0163350b 2024-11-16T10:15:59,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6534162d1894b79648a1282b0163350b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T10:15:59,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=A 2024-11-16T10:15:59,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:59,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=B 2024-11-16T10:15:59,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:59,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=C 2024-11-16T10:15:59,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:15:59,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c99c386473e246a696c296fca0cf36d5_6534162d1894b79648a1282b0163350b is 50, key is test_row_0/A:col10/1731752158921/Put/seqid=0 2024-11-16T10:15:59,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742227_1403 (size=14994) 2024-11-16T10:15:59,575 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:15:59,578 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c99c386473e246a696c296fca0cf36d5_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c99c386473e246a696c296fca0cf36d5_6534162d1894b79648a1282b0163350b 2024-11-16T10:15:59,579 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/e42fe3c22a974cab84570e5db18b276a, store: [table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:15:59,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/e42fe3c22a974cab84570e5db18b276a is 175, key is test_row_0/A:col10/1731752158921/Put/seqid=0 2024-11-16T10:15:59,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742228_1404 (size=39949) 2024-11-16T10:15:59,587 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=411, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/e42fe3c22a974cab84570e5db18b276a 2024-11-16T10:15:59,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/0e413cbd4100464f93771289a07bca5e is 50, key is test_row_0/B:col10/1731752158921/Put/seqid=0 2024-11-16T10:15:59,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742229_1405 (size=12301) 2024-11-16T10:15:59,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:59,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752219645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:59,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:59,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:59,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752219647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:59,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:59,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:59,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752219648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:59,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:59,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:59,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752219652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:59,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:59,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:59,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752219764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:59,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:59,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:59,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752219764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:59,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:59,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:59,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752219764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:59,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:59,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:59,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752219773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:59,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752219894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:59,905 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in 6534162d1894b79648a1282b0163350b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4265 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:15:59,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:59,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:59,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752219974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:59,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:59,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:59,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752219975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:59,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:15:59,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:15:59,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752219975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:15:59,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:15:59,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:00,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/0e413cbd4100464f93771289a07bca5e 2024-11-16T10:16:00,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:00,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752219994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:00,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:00,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:00,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:00,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/193bfb818e9543d8aa7fab69cf9a8cba is 50, key is test_row_0/C:col10/1731752158921/Put/seqid=0 2024-11-16T10:16:00,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742230_1406 (size=12301) 2024-11-16T10:16:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T10:16:00,095 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:00,095 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:16:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-16T10:16:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T10:16:00,098 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:16:00,098 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:16:00,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:16:00,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T10:16:00,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T10:16:00,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:00,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:16:00,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:00,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:00,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:00,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:00,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752220294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:00,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752220294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:00,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:00,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:00,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:00,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:00,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:00,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752220294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:00,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:00,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:00,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:00,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:00,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752220324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:00,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T10:16:00,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:00,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:16:00,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:00,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:00,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T10:16:00,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/193bfb818e9543d8aa7fab69cf9a8cba 2024-11-16T10:16:00,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/e42fe3c22a974cab84570e5db18b276a as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/e42fe3c22a974cab84570e5db18b276a 2024-11-16T10:16:00,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/e42fe3c22a974cab84570e5db18b276a, entries=200, sequenceid=411, filesize=39.0 K 2024-11-16T10:16:00,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/0e413cbd4100464f93771289a07bca5e as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/0e413cbd4100464f93771289a07bca5e 2024-11-16T10:16:00,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/0e413cbd4100464f93771289a07bca5e, entries=150, sequenceid=411, filesize=12.0 K 2024-11-16T10:16:00,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/193bfb818e9543d8aa7fab69cf9a8cba as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/193bfb818e9543d8aa7fab69cf9a8cba 2024-11-16T10:16:00,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/193bfb818e9543d8aa7fab69cf9a8cba, entries=150, sequenceid=411, filesize=12.0 K 2024-11-16T10:16:00,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6534162d1894b79648a1282b0163350b in 896ms, sequenceid=411, compaction requested=true 2024-11-16T10:16:00,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:16:00,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:16:00,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:00,462 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:16:00,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:16:00,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:00,462 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:16:00,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:16:00,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:00,463 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:16:00,463 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:16:00,463 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/B is initiating minor compaction (all files) 2024-11-16T10:16:00,463 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/A is initiating minor compaction (all files) 2024-11-16T10:16:00,463 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/B in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:00,463 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/A in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:00,463 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/234830f5b2ac4c5f9a5aa5c19bfc2ae7, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/7792c44bb0af4a6e8e7368ee38b8b117, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/5dc7378cc088424fa493ac369f935156, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/0e413cbd4100464f93771289a07bca5e] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=48.5 K 2024-11-16T10:16:00,463 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/fac80d73c20341de87c95f7b14f06df5, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/4190a7032a8a4dcd9975836dde31feb0, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/24823500d81e49b59a0dd51de84bb56a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/e42fe3c22a974cab84570e5db18b276a] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=139.5 K 2024-11-16T10:16:00,463 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:00,463 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. files: [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/fac80d73c20341de87c95f7b14f06df5, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/4190a7032a8a4dcd9975836dde31feb0, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/24823500d81e49b59a0dd51de84bb56a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/e42fe3c22a974cab84570e5db18b276a] 2024-11-16T10:16:00,463 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 234830f5b2ac4c5f9a5aa5c19bfc2ae7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731752155638 2024-11-16T10:16:00,463 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting fac80d73c20341de87c95f7b14f06df5, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731752155638 2024-11-16T10:16:00,463 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 7792c44bb0af4a6e8e7368ee38b8b117, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731752156294 2024-11-16T10:16:00,463 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4190a7032a8a4dcd9975836dde31feb0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731752156289 2024-11-16T10:16:00,464 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dc7378cc088424fa493ac369f935156, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731752157685 2024-11-16T10:16:00,464 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24823500d81e49b59a0dd51de84bb56a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731752157685 2024-11-16T10:16:00,464 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e413cbd4100464f93771289a07bca5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731752158921 2024-11-16T10:16:00,464 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting e42fe3c22a974cab84570e5db18b276a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731752158917 2024-11-16T10:16:00,479 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:16:00,480 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116ee5d3d50a9d447e7a744e282a96ca9fd_6534162d1894b79648a1282b0163350b store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:16:00,481 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#B#compaction#346 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:00,482 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/a61661372fdf4674b0ca26019213f97f is 50, key is test_row_0/B:col10/1731752158921/Put/seqid=0 2024-11-16T10:16:00,482 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116ee5d3d50a9d447e7a744e282a96ca9fd_6534162d1894b79648a1282b0163350b, store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:16:00,482 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ee5d3d50a9d447e7a744e282a96ca9fd_6534162d1894b79648a1282b0163350b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:16:00,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742231_1407 (size=12881) 2024-11-16T10:16:00,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742232_1408 (size=4469) 2024-11-16T10:16:00,500 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#A#compaction#345 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:00,501 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/6b9540cbab1b48548f831d05cd06fa43 is 175, key is test_row_0/A:col10/1731752158921/Put/seqid=0 2024-11-16T10:16:00,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742233_1409 (size=31835) 2024-11-16T10:16:00,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T10:16:00,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:00,561 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 6534162d1894b79648a1282b0163350b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T10:16:00,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=A 2024-11-16T10:16:00,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:00,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=B 2024-11-16T10:16:00,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:00,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=C 2024-11-16T10:16:00,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:00,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fc157365295d4260b9b2aec30f94e2b4_6534162d1894b79648a1282b0163350b is 50, key is test_row_0/A:col10/1731752159646/Put/seqid=0 2024-11-16T10:16:00,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742234_1410 (size=12454) 2024-11-16T10:16:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T10:16:00,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 6534162d1894b79648a1282b0163350b 2024-11-16T10:16:00,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:16:00,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:00,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:00,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752220858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:00,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752220858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:00,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:00,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:00,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:00,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:00,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:00,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:00,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752220870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:00,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:00,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:00,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:00,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752220870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:00,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:00,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:00,898 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/a61661372fdf4674b0ca26019213f97f as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/a61661372fdf4674b0ca26019213f97f 2024-11-16T10:16:00,907 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6534162d1894b79648a1282b0163350b/B of 6534162d1894b79648a1282b0163350b into a61661372fdf4674b0ca26019213f97f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:00,907 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:16:00,907 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/B, priority=12, startTime=1731752160462; duration=0sec 2024-11-16T10:16:00,907 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:00,907 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:B 2024-11-16T10:16:00,908 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:16:00,916 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:16:00,916 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/C is initiating minor compaction (all files) 2024-11-16T10:16:00,916 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/C in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:00,916 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/57b51273ebed4ac385c78587be1965ff, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/af69d20769d146c0bc3a3038833c5784, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/d60d01f8a3994950ad0de5ab74ef2c83, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/193bfb818e9543d8aa7fab69cf9a8cba] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=48.5 K 2024-11-16T10:16:00,917 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 57b51273ebed4ac385c78587be1965ff, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731752155638 2024-11-16T10:16:00,917 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting af69d20769d146c0bc3a3038833c5784, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731752156294 2024-11-16T10:16:00,917 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/6b9540cbab1b48548f831d05cd06fa43 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/6b9540cbab1b48548f831d05cd06fa43 2024-11-16T10:16:00,917 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting d60d01f8a3994950ad0de5ab74ef2c83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731752157685 2024-11-16T10:16:00,918 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 193bfb818e9543d8aa7fab69cf9a8cba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731752158921 2024-11-16T10:16:00,923 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6534162d1894b79648a1282b0163350b/A of 6534162d1894b79648a1282b0163350b into 6b9540cbab1b48548f831d05cd06fa43(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:00,923 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:16:00,923 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/A, priority=12, startTime=1731752160462; duration=0sec 2024-11-16T10:16:00,924 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:00,924 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:A 2024-11-16T10:16:00,929 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#C#compaction#348 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:00,929 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/b7e2b3a65a674427aac9216b6734a9de is 50, key is test_row_0/C:col10/1731752158921/Put/seqid=0 2024-11-16T10:16:00,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:00,976 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fc157365295d4260b9b2aec30f94e2b4_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fc157365295d4260b9b2aec30f94e2b4_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:00,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/27c45631e51c4c1584dfe110a47cb886, store: [table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:16:00,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/27c45631e51c4c1584dfe110a47cb886 is 175, key is test_row_0/A:col10/1731752159646/Put/seqid=0 2024-11-16T10:16:00,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:00,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752220974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:00,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:00,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:00,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:00,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742235_1411 (size=12881) 2024-11-16T10:16:00,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752220974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:00,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:00,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:00,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:01,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752221004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:01,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:01,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:01,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:01,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752221005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:01,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:01,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:01,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:01,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742236_1412 (size=31255) 2024-11-16T10:16:01,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752221194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:01,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:01,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:01,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:01,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752221206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:01,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:01,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:01,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:01,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752221225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:01,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:01,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:01,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:01,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752221225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T10:16:01,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:01,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:01,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:01,392 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/b7e2b3a65a674427aac9216b6734a9de as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/b7e2b3a65a674427aac9216b6734a9de 2024-11-16T10:16:01,395 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6534162d1894b79648a1282b0163350b/C of 6534162d1894b79648a1282b0163350b into b7e2b3a65a674427aac9216b6734a9de(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:01,395 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:16:01,395 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/C, priority=12, startTime=1731752160462; duration=0sec 2024-11-16T10:16:01,395 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:01,395 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:C 2024-11-16T10:16:01,437 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=431, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/27c45631e51c4c1584dfe110a47cb886 2024-11-16T10:16:01,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/5b46317b450d44ee95e3bf9150958568 is 50, key is test_row_0/B:col10/1731752159646/Put/seqid=0 2024-11-16T10:16:01,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742237_1413 (size=12301) 2024-11-16T10:16:01,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752221514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:01,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:01,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:01,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:01,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:01,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752221524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:01,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:01,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:01,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:01,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752221544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:01,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:01,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:01,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:01,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752221544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:01,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:01,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:01,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:01,857 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/5b46317b450d44ee95e3bf9150958568 2024-11-16T10:16:01,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/3680cc212f864e74944237be52f26a82 is 50, key is test_row_0/C:col10/1731752159646/Put/seqid=0 2024-11-16T10:16:01,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742238_1414 (size=12301) 2024-11-16T10:16:02,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752222034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:02,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:02,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:02,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752222034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:02,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:02,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:02,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:02,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:02,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752222054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:02,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752222054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:02,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T10:16:02,267 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/3680cc212f864e74944237be52f26a82 2024-11-16T10:16:02,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/27c45631e51c4c1584dfe110a47cb886 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/27c45631e51c4c1584dfe110a47cb886 2024-11-16T10:16:02,275 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/27c45631e51c4c1584dfe110a47cb886, entries=150, sequenceid=431, filesize=30.5 K 2024-11-16T10:16:02,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/5b46317b450d44ee95e3bf9150958568 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/5b46317b450d44ee95e3bf9150958568 2024-11-16T10:16:02,280 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/5b46317b450d44ee95e3bf9150958568, entries=150, sequenceid=431, filesize=12.0 K 2024-11-16T10:16:02,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/3680cc212f864e74944237be52f26a82 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/3680cc212f864e74944237be52f26a82 2024-11-16T10:16:02,288 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/3680cc212f864e74944237be52f26a82, entries=150, sequenceid=431, filesize=12.0 K 2024-11-16T10:16:02,290 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6534162d1894b79648a1282b0163350b in 1728ms, sequenceid=431, compaction requested=false 2024-11-16T10:16:02,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:16:02,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:02,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-16T10:16:02,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-16T10:16:02,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-16T10:16:02,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1930 sec 2024-11-16T10:16:02,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.1990 sec 2024-11-16T10:16:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 6534162d1894b79648a1282b0163350b 2024-11-16T10:16:03,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6534162d1894b79648a1282b0163350b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T10:16:03,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=A 2024-11-16T10:16:03,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:03,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=B 2024-11-16T10:16:03,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:03,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=C 2024-11-16T10:16:03,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:03,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f0df4f1f16134c4abfee51794df6a36d_6534162d1894b79648a1282b0163350b is 50, key is test_row_0/A:col10/1731752160869/Put/seqid=0 2024-11-16T10:16:03,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742239_1415 (size=14994) 2024-11-16T10:16:03,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752223087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:03,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752223098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:03,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:03,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752223099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:03,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:03,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:03,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:03,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752223100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:03,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:03,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752223204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:03,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:03,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752223213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:03,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:03,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752223214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:03,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:03,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752223214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:03,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:03,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752223414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:03,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:03,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752223423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:03,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:03,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752223423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:03,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:03,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752223424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:03,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:03,459 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:03,462 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f0df4f1f16134c4abfee51794df6a36d_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f0df4f1f16134c4abfee51794df6a36d_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:03,463 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/5a4a3e758fb944229f3f68075b243493, store: [table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:16:03,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/5a4a3e758fb944229f3f68075b243493 is 175, key is test_row_0/A:col10/1731752160869/Put/seqid=0 2024-11-16T10:16:03,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742240_1416 (size=39949) 2024-11-16T10:16:03,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:03,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752223724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:03,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:03,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:03,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752223734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:03,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:03,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:03,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752223734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:03,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:03,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752223735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:03,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:03,869 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=447, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/5a4a3e758fb944229f3f68075b243493 2024-11-16T10:16:03,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/467a14773d09419493e27406639e0418 is 50, key is test_row_0/B:col10/1731752160869/Put/seqid=0 2024-11-16T10:16:03,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742241_1417 (size=12301) 2024-11-16T10:16:03,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:03,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36984 deadline: 1731752223914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:03,923 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in 6534162d1894b79648a1282b0163350b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8283 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:16:03,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:03,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:04,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:04,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752224244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:04,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:04,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752224244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:04,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:04,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:04,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:04,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:04,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:04,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:04,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:04,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752224254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:04,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:04,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:04,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:04,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752224254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:04,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:04,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:04,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:04,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T10:16:04,274 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:04,274 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:16:04,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-16T10:16:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T10:16:04,276 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:16:04,276 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:16:04,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:16:04,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/467a14773d09419493e27406639e0418 2024-11-16T10:16:04,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/999f9ad4b21a4b57a07b64ae7806d437 is 50, key is test_row_0/C:col10/1731752160869/Put/seqid=0 2024-11-16T10:16:04,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742242_1418 (size=12301) 2024-11-16T10:16:04,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T10:16:04,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-16T10:16:04,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:04,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:16:04,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:04,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:04,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:04,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:04,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-16T10:16:04,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:04,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:16:04,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:04,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:04,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:04,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T10:16:04,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/999f9ad4b21a4b57a07b64ae7806d437 2024-11-16T10:16:04,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/5a4a3e758fb944229f3f68075b243493 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/5a4a3e758fb944229f3f68075b243493 2024-11-16T10:16:04,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/5a4a3e758fb944229f3f68075b243493, entries=200, sequenceid=447, filesize=39.0 K 2024-11-16T10:16:04,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/467a14773d09419493e27406639e0418 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/467a14773d09419493e27406639e0418 2024-11-16T10:16:04,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/467a14773d09419493e27406639e0418, entries=150, sequenceid=447, filesize=12.0 K 2024-11-16T10:16:04,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/999f9ad4b21a4b57a07b64ae7806d437 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/999f9ad4b21a4b57a07b64ae7806d437 2024-11-16T10:16:04,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/999f9ad4b21a4b57a07b64ae7806d437, entries=150, sequenceid=447, filesize=12.0 K 2024-11-16T10:16:04,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6534162d1894b79648a1282b0163350b in 1656ms, sequenceid=447, compaction requested=true 2024-11-16T10:16:04,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:16:04,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:16:04,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:04,705 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:04,705 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:04,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:16:04,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:04,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:16:04,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:04,709 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:04,709 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:04,709 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/A is initiating minor compaction (all files) 2024-11-16T10:16:04,709 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/B is initiating minor compaction (all files) 2024-11-16T10:16:04,709 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/A in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:04,709 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/B in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:04,709 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/a61661372fdf4674b0ca26019213f97f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/5b46317b450d44ee95e3bf9150958568, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/467a14773d09419493e27406639e0418] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=36.6 K 2024-11-16T10:16:04,709 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/6b9540cbab1b48548f831d05cd06fa43, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/27c45631e51c4c1584dfe110a47cb886, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/5a4a3e758fb944229f3f68075b243493] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=100.6 K 2024-11-16T10:16:04,709 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:04,709 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. files: [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/6b9540cbab1b48548f831d05cd06fa43, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/27c45631e51c4c1584dfe110a47cb886, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/5a4a3e758fb944229f3f68075b243493] 2024-11-16T10:16:04,709 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting a61661372fdf4674b0ca26019213f97f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731752158921 2024-11-16T10:16:04,709 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b9540cbab1b48548f831d05cd06fa43, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731752158921 2024-11-16T10:16:04,711 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27c45631e51c4c1584dfe110a47cb886, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731752159645 2024-11-16T10:16:04,712 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b46317b450d44ee95e3bf9150958568, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731752159645 2024-11-16T10:16:04,712 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a4a3e758fb944229f3f68075b243493, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731752160844 2024-11-16T10:16:04,712 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 467a14773d09419493e27406639e0418, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731752160868 2024-11-16T10:16:04,718 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:16:04,719 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#B#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:04,719 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/81204eec1a8c42b5aa0fafb647e0f251 is 50, key is test_row_0/B:col10/1731752160869/Put/seqid=0 2024-11-16T10:16:04,721 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111638ca2ce410b54855b87131cae79d18c3_6534162d1894b79648a1282b0163350b store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:16:04,723 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111638ca2ce410b54855b87131cae79d18c3_6534162d1894b79648a1282b0163350b, store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:16:04,724 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111638ca2ce410b54855b87131cae79d18c3_6534162d1894b79648a1282b0163350b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:16:04,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742243_1419 (size=12983) 2024-11-16T10:16:04,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-16T10:16:04,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:04,735 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 6534162d1894b79648a1282b0163350b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T10:16:04,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=A 2024-11-16T10:16:04,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:04,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=B 2024-11-16T10:16:04,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:04,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=C 2024-11-16T10:16:04,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:04,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742244_1420 (size=4469) 2024-11-16T10:16:04,744 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#A#compaction#355 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:04,744 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/0f7b6eefd6264c82913417d4a0796a86 is 175, key is test_row_0/A:col10/1731752160869/Put/seqid=0 2024-11-16T10:16:04,745 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/81204eec1a8c42b5aa0fafb647e0f251 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/81204eec1a8c42b5aa0fafb647e0f251 2024-11-16T10:16:04,750 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6534162d1894b79648a1282b0163350b/B of 6534162d1894b79648a1282b0163350b into 81204eec1a8c42b5aa0fafb647e0f251(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:04,750 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:16:04,751 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/B, priority=13, startTime=1731752164705; duration=0sec 2024-11-16T10:16:04,751 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:04,751 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:B 2024-11-16T10:16:04,751 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:04,752 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:04,752 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/C is initiating minor compaction (all files) 2024-11-16T10:16:04,752 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/C in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:04,752 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/b7e2b3a65a674427aac9216b6734a9de, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/3680cc212f864e74944237be52f26a82, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/999f9ad4b21a4b57a07b64ae7806d437] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=36.6 K 2024-11-16T10:16:04,753 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting b7e2b3a65a674427aac9216b6734a9de, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731752158921 2024-11-16T10:16:04,753 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 3680cc212f864e74944237be52f26a82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731752159645 2024-11-16T10:16:04,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d994fd3c9ad348089341bfceba3b0ab9_6534162d1894b79648a1282b0163350b is 50, key is test_row_0/A:col10/1731752163098/Put/seqid=0 2024-11-16T10:16:04,753 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 999f9ad4b21a4b57a07b64ae7806d437, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731752160868 2024-11-16T10:16:04,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742245_1421 (size=31937) 2024-11-16T10:16:04,802 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#C#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:04,803 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/20e76961b259474aa2f1da2ce9d7e780 is 50, key is test_row_0/C:col10/1731752160869/Put/seqid=0 2024-11-16T10:16:04,803 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/0f7b6eefd6264c82913417d4a0796a86 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/0f7b6eefd6264c82913417d4a0796a86 2024-11-16T10:16:04,812 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6534162d1894b79648a1282b0163350b/A of 6534162d1894b79648a1282b0163350b into 0f7b6eefd6264c82913417d4a0796a86(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:04,812 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:16:04,812 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/A, priority=13, startTime=1731752164705; duration=0sec 2024-11-16T10:16:04,812 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:04,812 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:A 2024-11-16T10:16:04,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742246_1422 (size=12454) 2024-11-16T10:16:04,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:04,832 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d994fd3c9ad348089341bfceba3b0ab9_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d994fd3c9ad348089341bfceba3b0ab9_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:04,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/9d27846ad75446f5832f8786b003593f, store: [table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:16:04,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/9d27846ad75446f5832f8786b003593f is 175, key is test_row_0/A:col10/1731752163098/Put/seqid=0 2024-11-16T10:16:04,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742247_1423 (size=12983) 2024-11-16T10:16:04,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742248_1424 (size=31255) 2024-11-16T10:16:04,844 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=466, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/9d27846ad75446f5832f8786b003593f 2024-11-16T10:16:04,849 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/20e76961b259474aa2f1da2ce9d7e780 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/20e76961b259474aa2f1da2ce9d7e780 2024-11-16T10:16:04,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/824284cee08c4ef4b6f77a899cf5a0e6 is 50, key is test_row_0/B:col10/1731752163098/Put/seqid=0 2024-11-16T10:16:04,861 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6534162d1894b79648a1282b0163350b/C of 6534162d1894b79648a1282b0163350b into 20e76961b259474aa2f1da2ce9d7e780(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:04,861 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:16:04,861 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/C, priority=13, startTime=1731752164705; duration=0sec 2024-11-16T10:16:04,861 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:04,861 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:C 2024-11-16T10:16:04,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742249_1425 (size=12301) 2024-11-16T10:16:04,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T10:16:05,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 6534162d1894b79648a1282b0163350b 2024-11-16T10:16:05,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:16:05,270 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/824284cee08c4ef4b6f77a899cf5a0e6 2024-11-16T10:16:05,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/e005dc1626fb48d984adbe0cbab594bd is 50, key is test_row_0/C:col10/1731752163098/Put/seqid=0 2024-11-16T10:16:05,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742250_1426 (size=12301) 2024-11-16T10:16:05,294 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/e005dc1626fb48d984adbe0cbab594bd 2024-11-16T10:16:05,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752225288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:05,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:05,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/9d27846ad75446f5832f8786b003593f as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/9d27846ad75446f5832f8786b003593f 2024-11-16T10:16:05,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752225295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:05,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:05,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752225298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:05,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752225299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:05,308 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/9d27846ad75446f5832f8786b003593f, entries=150, sequenceid=466, filesize=30.5 K 2024-11-16T10:16:05,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/824284cee08c4ef4b6f77a899cf5a0e6 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/824284cee08c4ef4b6f77a899cf5a0e6 2024-11-16T10:16:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:05,316 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/824284cee08c4ef4b6f77a899cf5a0e6, entries=150, sequenceid=466, filesize=12.0 K 2024-11-16T10:16:05,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/e005dc1626fb48d984adbe0cbab594bd as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/e005dc1626fb48d984adbe0cbab594bd 2024-11-16T10:16:05,322 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/e005dc1626fb48d984adbe0cbab594bd, entries=150, sequenceid=466, filesize=12.0 K 2024-11-16T10:16:05,323 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6534162d1894b79648a1282b0163350b in 589ms, sequenceid=466, compaction requested=false 2024-11-16T10:16:05,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:16:05,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:05,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-16T10:16:05,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-16T10:16:05,327 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-16T10:16:05,327 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0490 sec 2024-11-16T10:16:05,333 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.0540 sec 2024-11-16T10:16:05,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6534162d1894b79648a1282b0163350b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T10:16:05,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=A 2024-11-16T10:16:05,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:05,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=B 2024-11-16T10:16:05,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:05,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=C 2024-11-16T10:16:05,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 6534162d1894b79648a1282b0163350b 2024-11-16T10:16:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T10:16:05,415 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:05,415 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:16:05,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-16T10:16:05,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T10:16:05,418 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:16:05,418 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:16:05,419 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:16:05,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b79c76e8cd51450b924c22767232fbd1_6534162d1894b79648a1282b0163350b is 50, key is test_row_0/A:col10/1731752165409/Put/seqid=0 2024-11-16T10:16:05,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742251_1427 (size=14994) 2024-11-16T10:16:05,454 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:05,458 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b79c76e8cd51450b924c22767232fbd1_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b79c76e8cd51450b924c22767232fbd1_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:05,459 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/ce806ce5d894457eb30959da270126a2, store: [table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:16:05,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/ce806ce5d894457eb30959da270126a2 is 175, key is test_row_0/A:col10/1731752165409/Put/seqid=0 2024-11-16T10:16:05,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742252_1428 (size=39949) 2024-11-16T10:16:05,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752225463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:05,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:05,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752225483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:05,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752225486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:05,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:05,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:05,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752225488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:05,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T10:16:05,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-16T10:16:05,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:05,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:16:05,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:05,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:05,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:05,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752225594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:05,596 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:05,596 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,596 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752225594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:05,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:05,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752225604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:05,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:05,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752225613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:05,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:05,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-16T10:16:05,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:05,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:16:05,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:05,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:05,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T10:16:05,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752225804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:05,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752225804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:05,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:05,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752225823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:05,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:05,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752225834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:05,870 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=483, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/ce806ce5d894457eb30959da270126a2 2024-11-16T10:16:05,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/c3b3662725fb44a2ac4c85eacdb691c9 is 50, key is test_row_0/B:col10/1731752165409/Put/seqid=0 2024-11-16T10:16:05,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-16T10:16:05,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:05,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:16:05,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:05,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:05,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:05,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742253_1429 (size=12301) 2024-11-16T10:16:05,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/c3b3662725fb44a2ac4c85eacdb691c9 2024-11-16T10:16:05,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/643ed8d082c74da3b2e2e8a8893a19db is 50, key is test_row_0/C:col10/1731752165409/Put/seqid=0 2024-11-16T10:16:05,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742254_1430 (size=12301) 2024-11-16T10:16:06,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-16T10:16:06,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:06,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:16:06,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:06,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:06,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T10:16:06,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752226114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:06,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:06,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752226124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:06,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:06,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752226144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:06,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752226154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:06,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:06,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-16T10:16:06,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:06,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:16:06,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:06,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:06,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:06,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/643ed8d082c74da3b2e2e8a8893a19db 2024-11-16T10:16:06,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/ce806ce5d894457eb30959da270126a2 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/ce806ce5d894457eb30959da270126a2 2024-11-16T10:16:06,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/ce806ce5d894457eb30959da270126a2, entries=200, sequenceid=483, filesize=39.0 K 2024-11-16T10:16:06,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/c3b3662725fb44a2ac4c85eacdb691c9 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/c3b3662725fb44a2ac4c85eacdb691c9 2024-11-16T10:16:06,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/c3b3662725fb44a2ac4c85eacdb691c9, entries=150, sequenceid=483, filesize=12.0 K 2024-11-16T10:16:06,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/643ed8d082c74da3b2e2e8a8893a19db as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/643ed8d082c74da3b2e2e8a8893a19db 2024-11-16T10:16:06,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/643ed8d082c74da3b2e2e8a8893a19db, entries=150, sequenceid=483, filesize=12.0 K 2024-11-16T10:16:06,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=114.05 KB/116790 for 6534162d1894b79648a1282b0163350b in 910ms, sequenceid=483, compaction requested=true 2024-11-16T10:16:06,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:16:06,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:16:06,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:06,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:16:06,322 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:06,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:06,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:16:06,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:06,322 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:06,323 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:06,323 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/B is initiating minor compaction (all files) 2024-11-16T10:16:06,323 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/B in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:06,323 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/81204eec1a8c42b5aa0fafb647e0f251, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/824284cee08c4ef4b6f77a899cf5a0e6, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/c3b3662725fb44a2ac4c85eacdb691c9] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=36.7 K 2024-11-16T10:16:06,323 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:06,323 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/A is initiating minor compaction (all files) 2024-11-16T10:16:06,323 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/A in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:06,323 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/0f7b6eefd6264c82913417d4a0796a86, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/9d27846ad75446f5832f8786b003593f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/ce806ce5d894457eb30959da270126a2] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=100.7 K 2024-11-16T10:16:06,323 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:06,323 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. files: [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/0f7b6eefd6264c82913417d4a0796a86, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/9d27846ad75446f5832f8786b003593f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/ce806ce5d894457eb30959da270126a2] 2024-11-16T10:16:06,324 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 81204eec1a8c42b5aa0fafb647e0f251, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731752160868 2024-11-16T10:16:06,324 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f7b6eefd6264c82913417d4a0796a86, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731752160868 2024-11-16T10:16:06,324 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d27846ad75446f5832f8786b003593f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731752163075 2024-11-16T10:16:06,324 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 824284cee08c4ef4b6f77a899cf5a0e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731752163075 2024-11-16T10:16:06,324 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting c3b3662725fb44a2ac4c85eacdb691c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731752165282 2024-11-16T10:16:06,324 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce806ce5d894457eb30959da270126a2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731752165282 2024-11-16T10:16:06,329 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:16:06,330 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111688df251aea6a49d0b38d26f6d4dcdfcc_6534162d1894b79648a1282b0163350b store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:16:06,330 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#B#compaction#364 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:06,331 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/5fd08f875ae44a22a2aaa57d19217765 is 50, key is test_row_0/B:col10/1731752165409/Put/seqid=0 2024-11-16T10:16:06,333 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111688df251aea6a49d0b38d26f6d4dcdfcc_6534162d1894b79648a1282b0163350b, store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:16:06,333 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111688df251aea6a49d0b38d26f6d4dcdfcc_6534162d1894b79648a1282b0163350b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:16:06,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-16T10:16:06,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:06,337 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 6534162d1894b79648a1282b0163350b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T10:16:06,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=A 2024-11-16T10:16:06,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:06,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=B 2024-11-16T10:16:06,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:06,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=C 2024-11-16T10:16:06,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:06,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742256_1432 (size=4469) 2024-11-16T10:16:06,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742255_1431 (size=13085) 2024-11-16T10:16:06,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116219057d193c442f99f0f48b0dbff2476_6534162d1894b79648a1282b0163350b is 50, key is test_row_0/A:col10/1731752165481/Put/seqid=0 2024-11-16T10:16:06,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742257_1433 (size=12454) 2024-11-16T10:16:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T10:16:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 6534162d1894b79648a1282b0163350b 2024-11-16T10:16:06,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. as already flushing 2024-11-16T10:16:06,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752226675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:06,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:06,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:06,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752226713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:06,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752226713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:06,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752226713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:06,747 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#A#compaction#363 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:06,747 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/09be4eec2950478ea8b98f70bab4aaf1 is 175, key is test_row_0/A:col10/1731752165409/Put/seqid=0 2024-11-16T10:16:06,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742258_1434 (size=32039) 2024-11-16T10:16:06,755 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/5fd08f875ae44a22a2aaa57d19217765 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/5fd08f875ae44a22a2aaa57d19217765 2024-11-16T10:16:06,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:06,760 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6534162d1894b79648a1282b0163350b/B of 6534162d1894b79648a1282b0163350b into 5fd08f875ae44a22a2aaa57d19217765(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:06,760 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:16:06,760 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/B, priority=13, startTime=1731752166322; duration=0sec 2024-11-16T10:16:06,760 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:06,760 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:B 2024-11-16T10:16:06,760 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116219057d193c442f99f0f48b0dbff2476_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116219057d193c442f99f0f48b0dbff2476_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:06,760 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:06,761 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:06,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/709c0a86842a4dcaa6711f689aae26ba, store: [table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:16:06,761 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/C is initiating minor compaction (all files) 2024-11-16T10:16:06,761 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/C in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:06,761 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/20e76961b259474aa2f1da2ce9d7e780, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/e005dc1626fb48d984adbe0cbab594bd, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/643ed8d082c74da3b2e2e8a8893a19db] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=36.7 K 2024-11-16T10:16:06,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/709c0a86842a4dcaa6711f689aae26ba is 175, key is test_row_0/A:col10/1731752165481/Put/seqid=0 2024-11-16T10:16:06,761 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 20e76961b259474aa2f1da2ce9d7e780, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731752160868 2024-11-16T10:16:06,762 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting e005dc1626fb48d984adbe0cbab594bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731752163075 2024-11-16T10:16:06,762 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 643ed8d082c74da3b2e2e8a8893a19db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731752165282 2024-11-16T10:16:06,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742259_1435 (size=31255) 2024-11-16T10:16:06,765 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=503, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/709c0a86842a4dcaa6711f689aae26ba 2024-11-16T10:16:06,770 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#C#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:06,770 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/059283d1cb3a4f64b8dd59725d25377a is 50, key is test_row_0/C:col10/1731752165409/Put/seqid=0 2024-11-16T10:16:06,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/fb7652a8350d472ca172af895fe0047f is 50, key is test_row_0/B:col10/1731752165481/Put/seqid=0 2024-11-16T10:16:06,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742260_1436 (size=13085) 2024-11-16T10:16:06,786 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/059283d1cb3a4f64b8dd59725d25377a as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/059283d1cb3a4f64b8dd59725d25377a 2024-11-16T10:16:06,790 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6534162d1894b79648a1282b0163350b/C of 6534162d1894b79648a1282b0163350b into 059283d1cb3a4f64b8dd59725d25377a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:06,790 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:16:06,790 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/C, priority=13, startTime=1731752166322; duration=0sec 2024-11-16T10:16:06,790 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:06,790 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:C 2024-11-16T10:16:06,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742261_1437 (size=12301) 2024-11-16T10:16:06,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:06,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752226824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:06,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752226833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752226834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:06,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752226834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:06,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:07,028 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T10:16:07,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752227034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:07,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:07,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:07,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:07,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752227044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:07,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752227044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:07,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752227044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:07,154 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/09be4eec2950478ea8b98f70bab4aaf1 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/09be4eec2950478ea8b98f70bab4aaf1 2024-11-16T10:16:07,158 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6534162d1894b79648a1282b0163350b/A of 6534162d1894b79648a1282b0163350b into 09be4eec2950478ea8b98f70bab4aaf1(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:07,158 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:16:07,158 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/A, priority=13, startTime=1731752166322; duration=0sec 2024-11-16T10:16:07,159 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:07,159 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:A 2024-11-16T10:16:07,193 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/fb7652a8350d472ca172af895fe0047f 2024-11-16T10:16:07,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/505dfa6435a540bea3cdf4ab9fd1940a is 50, key is test_row_0/C:col10/1731752165481/Put/seqid=0 2024-11-16T10:16:07,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742262_1438 (size=12301) 2024-11-16T10:16:07,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1731752227344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:07,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:07,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:07,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:07,355 INFO [Thread-1557 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1557. 2024-11-16T10:16:07,355 DEBUG [Thread-1557 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:16:07,355 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:07,356 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:07,356 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:16:07,359 INFO [Thread-1553 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1553. 2024-11-16T10:16:07,359 DEBUG [Thread-1553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:16:07,359 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:07,359 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:07,359 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:16:07,362 INFO [Thread-1559 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1559. 2024-11-16T10:16:07,362 DEBUG [Thread-1559 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:16:07,362 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:07,362 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:07,362 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:16:07,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37026 deadline: 1731752227364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:07,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37010 deadline: 1731752227364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:07,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36970 deadline: 1731752227364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:07,365 INFO [Thread-1551 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1551. 2024-11-16T10:16:07,365 DEBUG [Thread-1551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:16:07,365 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:07,365 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:07,365 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:16:07,369 INFO [Thread-1555 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1555. 2024-11-16T10:16:07,369 DEBUG [Thread-1555 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:16:07,369 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:07,369 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:07,370 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:16:07,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T10:16:07,602 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/505dfa6435a540bea3cdf4ab9fd1940a 2024-11-16T10:16:07,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/709c0a86842a4dcaa6711f689aae26ba as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/709c0a86842a4dcaa6711f689aae26ba 2024-11-16T10:16:07,609 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/709c0a86842a4dcaa6711f689aae26ba, entries=150, sequenceid=503, filesize=30.5 K 2024-11-16T10:16:07,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/fb7652a8350d472ca172af895fe0047f as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/fb7652a8350d472ca172af895fe0047f 2024-11-16T10:16:07,613 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/fb7652a8350d472ca172af895fe0047f, entries=150, sequenceid=503, filesize=12.0 K 2024-11-16T10:16:07,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/505dfa6435a540bea3cdf4ab9fd1940a as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/505dfa6435a540bea3cdf4ab9fd1940a 2024-11-16T10:16:07,617 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/505dfa6435a540bea3cdf4ab9fd1940a, entries=150, sequenceid=503, filesize=12.0 K 2024-11-16T10:16:07,618 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=73.80 KB/75570 for 6534162d1894b79648a1282b0163350b in 1280ms, sequenceid=503, compaction requested=false 2024-11-16T10:16:07,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:16:07,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:07,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-16T10:16:07,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-16T10:16:07,620 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-16T10:16:07,620 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2000 sec 2024-11-16T10:16:07,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.2050 sec 2024-11-16T10:16:07,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 6534162d1894b79648a1282b0163350b 2024-11-16T10:16:07,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6534162d1894b79648a1282b0163350b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-16T10:16:07,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=A 2024-11-16T10:16:07,855 INFO [Thread-1548 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1548. 2024-11-16T10:16:07,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:07,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=B 2024-11-16T10:16:07,855 DEBUG [Thread-1548 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:16:07,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:07,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=C 2024-11-16T10:16:07,855 DEBUG [Thread-1548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:07,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:07,856 DEBUG [Thread-1548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:07,856 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:16:07,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167c5127d515f84ec3af73e71d02a52489_6534162d1894b79648a1282b0163350b is 50, key is test_row_0/A:col10/1731752167854/Put/seqid=0 2024-11-16T10:16:07,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742263_1439 (size=12454) 2024-11-16T10:16:07,876 INFO [Thread-1544 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1544. 2024-11-16T10:16:07,876 INFO [Thread-1546 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1546. 2024-11-16T10:16:07,876 INFO [Thread-1540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1540. 2024-11-16T10:16:07,876 DEBUG [Thread-1544 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:16:07,876 DEBUG [Thread-1540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:16:07,876 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:07,876 DEBUG [Thread-1546 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:16:07,876 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:07,876 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:07,876 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:07,876 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:07,876 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:07,876 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:16:07,876 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:16:07,876 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:16:08,267 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:08,271 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167c5127d515f84ec3af73e71d02a52489_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167c5127d515f84ec3af73e71d02a52489_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:08,272 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/81b577dd3a524e14b4c2566d19d4d960, store: [table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:16:08,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/81b577dd3a524e14b4c2566d19d4d960 is 175, key is test_row_0/A:col10/1731752167854/Put/seqid=0 2024-11-16T10:16:08,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742264_1440 (size=31255) 2024-11-16T10:16:08,678 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=521, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/81b577dd3a524e14b4c2566d19d4d960 2024-11-16T10:16:08,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/6024a0b701d441449e532f5757417f30 is 50, key is test_row_0/B:col10/1731752167854/Put/seqid=0 2024-11-16T10:16:08,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742265_1441 (size=12301) 2024-11-16T10:16:09,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/6024a0b701d441449e532f5757417f30 2024-11-16T10:16:09,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/0b1389423fe9406b9879390042f72b83 is 50, key is test_row_0/C:col10/1731752167854/Put/seqid=0 2024-11-16T10:16:09,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742266_1442 (size=12301) 2024-11-16T10:16:09,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/0b1389423fe9406b9879390042f72b83 2024-11-16T10:16:09,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/81b577dd3a524e14b4c2566d19d4d960 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/81b577dd3a524e14b4c2566d19d4d960 2024-11-16T10:16:09,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/81b577dd3a524e14b4c2566d19d4d960, entries=150, sequenceid=521, filesize=30.5 K 2024-11-16T10:16:09,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/6024a0b701d441449e532f5757417f30 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/6024a0b701d441449e532f5757417f30 2024-11-16T10:16:09,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/6024a0b701d441449e532f5757417f30, entries=150, sequenceid=521, filesize=12.0 K 2024-11-16T10:16:09,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/0b1389423fe9406b9879390042f72b83 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/0b1389423fe9406b9879390042f72b83 2024-11-16T10:16:09,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/0b1389423fe9406b9879390042f72b83, entries=150, sequenceid=521, filesize=12.0 K 2024-11-16T10:16:09,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=20.13 KB/20610 for 6534162d1894b79648a1282b0163350b in 1674ms, sequenceid=521, compaction requested=true 2024-11-16T10:16:09,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:16:09,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:16:09,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:09,530 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:09,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:16:09,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:09,530 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:09,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6534162d1894b79648a1282b0163350b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:16:09,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:09,530 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:09,531 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:09,531 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/A is initiating minor compaction (all files) 2024-11-16T10:16:09,531 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/B is initiating minor compaction (all files) 2024-11-16T10:16:09,531 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/A in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:09,531 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/B in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:09,531 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/09be4eec2950478ea8b98f70bab4aaf1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/709c0a86842a4dcaa6711f689aae26ba, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/81b577dd3a524e14b4c2566d19d4d960] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=92.3 K 2024-11-16T10:16:09,531 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/5fd08f875ae44a22a2aaa57d19217765, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/fb7652a8350d472ca172af895fe0047f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/6024a0b701d441449e532f5757417f30] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=36.8 K 2024-11-16T10:16:09,531 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:09,531 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. files: [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/09be4eec2950478ea8b98f70bab4aaf1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/709c0a86842a4dcaa6711f689aae26ba, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/81b577dd3a524e14b4c2566d19d4d960] 2024-11-16T10:16:09,531 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fd08f875ae44a22a2aaa57d19217765, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731752165282 2024-11-16T10:16:09,531 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09be4eec2950478ea8b98f70bab4aaf1, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731752165282 2024-11-16T10:16:09,531 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting fb7652a8350d472ca172af895fe0047f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1731752165437 2024-11-16T10:16:09,531 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 709c0a86842a4dcaa6711f689aae26ba, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1731752165437 2024-11-16T10:16:09,531 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 6024a0b701d441449e532f5757417f30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1731752166689 2024-11-16T10:16:09,531 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81b577dd3a524e14b4c2566d19d4d960, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1731752166689 2024-11-16T10:16:09,536 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:16:09,537 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#B#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:09,537 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116e3b30291d4a244dba9c803123b3ceb0e_6534162d1894b79648a1282b0163350b store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:16:09,537 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/11b4da8d4b4a4572969ec5fa17c78e79 is 50, key is test_row_0/B:col10/1731752167854/Put/seqid=0 2024-11-16T10:16:09,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742267_1443 (size=13187) 2024-11-16T10:16:09,545 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116e3b30291d4a244dba9c803123b3ceb0e_6534162d1894b79648a1282b0163350b, store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:16:09,545 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e3b30291d4a244dba9c803123b3ceb0e_6534162d1894b79648a1282b0163350b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:16:09,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742268_1444 (size=4469) 2024-11-16T10:16:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T10:16:09,575 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:09,945 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/11b4da8d4b4a4572969ec5fa17c78e79 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/11b4da8d4b4a4572969ec5fa17c78e79 2024-11-16T10:16:09,949 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#A#compaction#373 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:09,949 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6534162d1894b79648a1282b0163350b/B of 6534162d1894b79648a1282b0163350b into 11b4da8d4b4a4572969ec5fa17c78e79(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:09,949 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:16:09,949 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/B, priority=13, startTime=1731752169530; duration=0sec 2024-11-16T10:16:09,949 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:09,949 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:B 2024-11-16T10:16:09,949 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:09,949 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/aec6de8b590b4b498e7beeaeaec4718a is 175, key is test_row_0/A:col10/1731752167854/Put/seqid=0 2024-11-16T10:16:09,950 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:09,950 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 6534162d1894b79648a1282b0163350b/C is initiating minor compaction (all files) 2024-11-16T10:16:09,950 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6534162d1894b79648a1282b0163350b/C in TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:09,950 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/059283d1cb3a4f64b8dd59725d25377a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/505dfa6435a540bea3cdf4ab9fd1940a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/0b1389423fe9406b9879390042f72b83] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp, totalSize=36.8 K 2024-11-16T10:16:09,950 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 059283d1cb3a4f64b8dd59725d25377a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731752165282 2024-11-16T10:16:09,951 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 505dfa6435a540bea3cdf4ab9fd1940a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1731752165437 2024-11-16T10:16:09,951 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b1389423fe9406b9879390042f72b83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1731752166689 2024-11-16T10:16:09,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742269_1445 (size=32141) 2024-11-16T10:16:09,958 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6534162d1894b79648a1282b0163350b#C#compaction#374 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:09,958 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/b44c8aa7db7e4e858bba0d5f023f9df3 is 50, key is test_row_0/C:col10/1731752167854/Put/seqid=0 2024-11-16T10:16:09,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742270_1446 (size=13187) 2024-11-16T10:16:10,363 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/aec6de8b590b4b498e7beeaeaec4718a as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/aec6de8b590b4b498e7beeaeaec4718a 2024-11-16T10:16:10,367 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/b44c8aa7db7e4e858bba0d5f023f9df3 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/b44c8aa7db7e4e858bba0d5f023f9df3 2024-11-16T10:16:10,367 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6534162d1894b79648a1282b0163350b/A of 6534162d1894b79648a1282b0163350b into aec6de8b590b4b498e7beeaeaec4718a(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:10,368 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:16:10,368 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/A, priority=13, startTime=1731752169530; duration=0sec 2024-11-16T10:16:10,368 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:10,368 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:A 2024-11-16T10:16:10,371 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6534162d1894b79648a1282b0163350b/C of 6534162d1894b79648a1282b0163350b into b44c8aa7db7e4e858bba0d5f023f9df3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:10,371 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6534162d1894b79648a1282b0163350b: 2024-11-16T10:16:10,371 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., storeName=6534162d1894b79648a1282b0163350b/C, priority=13, startTime=1731752169530; duration=0sec 2024-11-16T10:16:10,371 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:10,371 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6534162d1894b79648a1282b0163350b:C 2024-11-16T10:16:12,105 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/e829c5a1bd2b4bea9f422d6833ec6aa0, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/6c6d61ccc9a8457fb56a467f4a9de079, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/f8d2e7a0906a461fb1ce3f9705218436, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/3ad50cbae15d41fe8265a7992fff9056, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/f6490f78334d4fb38b25b75647850b55, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/7b38cfb2cd8d40b4b61fad7834ef1370, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/da4597c79f1e407389cae00d601dd268, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/4ad1e48a969c40268ccea858ac4523d0, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/8d4e60376b8f4717a1101210e722da20, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/66f6b2c9540c421da8f2c9947c1aca19, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/4fe34563e7ee4cac9a50e9821105fbb5, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/1d43aa79fbc5455bbc4bc25a9979cccb, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/5bc94f196ff34e0992ed00c8f3f9e375, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/fac80d73c20341de87c95f7b14f06df5, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/4190a7032a8a4dcd9975836dde31feb0, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/24823500d81e49b59a0dd51de84bb56a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/e42fe3c22a974cab84570e5db18b276a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/6b9540cbab1b48548f831d05cd06fa43, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/27c45631e51c4c1584dfe110a47cb886, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/5a4a3e758fb944229f3f68075b243493, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/0f7b6eefd6264c82913417d4a0796a86, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/9d27846ad75446f5832f8786b003593f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/ce806ce5d894457eb30959da270126a2, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/09be4eec2950478ea8b98f70bab4aaf1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/709c0a86842a4dcaa6711f689aae26ba, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/81b577dd3a524e14b4c2566d19d4d960] to archive 2024-11-16T10:16:12,106 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:16:12,108 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/e829c5a1bd2b4bea9f422d6833ec6aa0 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/e829c5a1bd2b4bea9f422d6833ec6aa0 2024-11-16T10:16:12,110 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/6c6d61ccc9a8457fb56a467f4a9de079 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/6c6d61ccc9a8457fb56a467f4a9de079 2024-11-16T10:16:12,111 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/f8d2e7a0906a461fb1ce3f9705218436 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/f8d2e7a0906a461fb1ce3f9705218436 2024-11-16T10:16:12,113 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/3ad50cbae15d41fe8265a7992fff9056 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/3ad50cbae15d41fe8265a7992fff9056 2024-11-16T10:16:12,114 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/f6490f78334d4fb38b25b75647850b55 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/f6490f78334d4fb38b25b75647850b55 2024-11-16T10:16:12,116 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/7b38cfb2cd8d40b4b61fad7834ef1370 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/7b38cfb2cd8d40b4b61fad7834ef1370 2024-11-16T10:16:12,117 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/da4597c79f1e407389cae00d601dd268 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/da4597c79f1e407389cae00d601dd268 2024-11-16T10:16:12,119 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/4ad1e48a969c40268ccea858ac4523d0 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/4ad1e48a969c40268ccea858ac4523d0 2024-11-16T10:16:12,120 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/8d4e60376b8f4717a1101210e722da20 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/8d4e60376b8f4717a1101210e722da20 2024-11-16T10:16:12,121 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/66f6b2c9540c421da8f2c9947c1aca19 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/66f6b2c9540c421da8f2c9947c1aca19 2024-11-16T10:16:12,122 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/4fe34563e7ee4cac9a50e9821105fbb5 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/4fe34563e7ee4cac9a50e9821105fbb5 2024-11-16T10:16:12,124 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/1d43aa79fbc5455bbc4bc25a9979cccb to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/1d43aa79fbc5455bbc4bc25a9979cccb 2024-11-16T10:16:12,125 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/5bc94f196ff34e0992ed00c8f3f9e375 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/5bc94f196ff34e0992ed00c8f3f9e375 2024-11-16T10:16:12,126 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/fac80d73c20341de87c95f7b14f06df5 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/fac80d73c20341de87c95f7b14f06df5 2024-11-16T10:16:12,128 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/4190a7032a8a4dcd9975836dde31feb0 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/4190a7032a8a4dcd9975836dde31feb0 2024-11-16T10:16:12,129 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/24823500d81e49b59a0dd51de84bb56a to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/24823500d81e49b59a0dd51de84bb56a 2024-11-16T10:16:12,131 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/e42fe3c22a974cab84570e5db18b276a to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/e42fe3c22a974cab84570e5db18b276a 2024-11-16T10:16:12,132 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/6b9540cbab1b48548f831d05cd06fa43 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/6b9540cbab1b48548f831d05cd06fa43 2024-11-16T10:16:12,133 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/27c45631e51c4c1584dfe110a47cb886 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/27c45631e51c4c1584dfe110a47cb886 2024-11-16T10:16:12,134 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/5a4a3e758fb944229f3f68075b243493 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/5a4a3e758fb944229f3f68075b243493 2024-11-16T10:16:12,135 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/0f7b6eefd6264c82913417d4a0796a86 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/0f7b6eefd6264c82913417d4a0796a86 2024-11-16T10:16:12,137 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/9d27846ad75446f5832f8786b003593f to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/9d27846ad75446f5832f8786b003593f 2024-11-16T10:16:12,138 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/ce806ce5d894457eb30959da270126a2 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/ce806ce5d894457eb30959da270126a2 2024-11-16T10:16:12,139 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/09be4eec2950478ea8b98f70bab4aaf1 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/09be4eec2950478ea8b98f70bab4aaf1 2024-11-16T10:16:12,140 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/709c0a86842a4dcaa6711f689aae26ba to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/709c0a86842a4dcaa6711f689aae26ba 2024-11-16T10:16:12,141 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/81b577dd3a524e14b4c2566d19d4d960 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/81b577dd3a524e14b4c2566d19d4d960 2024-11-16T10:16:12,147 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/1c339bfbd4e84479b8509503c86b95cd, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/3ac1ff9c083d4491bc84eb057c1f2a03, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/3fb0101739d14a54859011d134b7a4c8, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/12a8133be48841a4bb5591c4eac582b1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/ce21ac41730b499a94586b92ec74cad8, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/fc0779b78b0c42ceab2d594e8f69950b, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/5ca45a6f22764bde8ea347110a71240d, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/b6bd75e09ba4483a8edf7ce274dd4e3d, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/b2fbcf4acb564ffc90b0986b2e6a4e37, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/36d749fc3c714b51871f13da11cc5603, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/6b6f5bd33f2e421286f10ea0a1bbe96b, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/9ec865232f5640c69d9e0c6de574ecec, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/234830f5b2ac4c5f9a5aa5c19bfc2ae7, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/089a507ad5144e92830f86f0cc2fc276, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/7792c44bb0af4a6e8e7368ee38b8b117, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/5dc7378cc088424fa493ac369f935156, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/a61661372fdf4674b0ca26019213f97f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/0e413cbd4100464f93771289a07bca5e, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/5b46317b450d44ee95e3bf9150958568, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/81204eec1a8c42b5aa0fafb647e0f251, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/467a14773d09419493e27406639e0418, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/824284cee08c4ef4b6f77a899cf5a0e6, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/5fd08f875ae44a22a2aaa57d19217765, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/c3b3662725fb44a2ac4c85eacdb691c9, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/fb7652a8350d472ca172af895fe0047f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/6024a0b701d441449e532f5757417f30] to archive 2024-11-16T10:16:12,148 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:16:12,150 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/1c339bfbd4e84479b8509503c86b95cd to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/1c339bfbd4e84479b8509503c86b95cd 2024-11-16T10:16:12,152 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/3ac1ff9c083d4491bc84eb057c1f2a03 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/3ac1ff9c083d4491bc84eb057c1f2a03 2024-11-16T10:16:12,153 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/3fb0101739d14a54859011d134b7a4c8 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/3fb0101739d14a54859011d134b7a4c8 2024-11-16T10:16:12,154 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/12a8133be48841a4bb5591c4eac582b1 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/12a8133be48841a4bb5591c4eac582b1 2024-11-16T10:16:12,155 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/ce21ac41730b499a94586b92ec74cad8 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/ce21ac41730b499a94586b92ec74cad8 2024-11-16T10:16:12,156 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/fc0779b78b0c42ceab2d594e8f69950b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/fc0779b78b0c42ceab2d594e8f69950b 2024-11-16T10:16:12,157 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/5ca45a6f22764bde8ea347110a71240d to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/5ca45a6f22764bde8ea347110a71240d 2024-11-16T10:16:12,158 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/b6bd75e09ba4483a8edf7ce274dd4e3d to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/b6bd75e09ba4483a8edf7ce274dd4e3d 2024-11-16T10:16:12,159 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/b2fbcf4acb564ffc90b0986b2e6a4e37 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/b2fbcf4acb564ffc90b0986b2e6a4e37 2024-11-16T10:16:12,159 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/36d749fc3c714b51871f13da11cc5603 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/36d749fc3c714b51871f13da11cc5603 2024-11-16T10:16:12,160 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/6b6f5bd33f2e421286f10ea0a1bbe96b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/6b6f5bd33f2e421286f10ea0a1bbe96b 2024-11-16T10:16:12,161 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/9ec865232f5640c69d9e0c6de574ecec to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/9ec865232f5640c69d9e0c6de574ecec 2024-11-16T10:16:12,162 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/234830f5b2ac4c5f9a5aa5c19bfc2ae7 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/234830f5b2ac4c5f9a5aa5c19bfc2ae7 2024-11-16T10:16:12,163 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/089a507ad5144e92830f86f0cc2fc276 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/089a507ad5144e92830f86f0cc2fc276 2024-11-16T10:16:12,164 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/7792c44bb0af4a6e8e7368ee38b8b117 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/7792c44bb0af4a6e8e7368ee38b8b117 2024-11-16T10:16:12,165 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/5dc7378cc088424fa493ac369f935156 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/5dc7378cc088424fa493ac369f935156 2024-11-16T10:16:12,166 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/a61661372fdf4674b0ca26019213f97f to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/a61661372fdf4674b0ca26019213f97f 2024-11-16T10:16:12,167 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/0e413cbd4100464f93771289a07bca5e to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/0e413cbd4100464f93771289a07bca5e 2024-11-16T10:16:12,169 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/5b46317b450d44ee95e3bf9150958568 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/5b46317b450d44ee95e3bf9150958568 2024-11-16T10:16:12,170 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/81204eec1a8c42b5aa0fafb647e0f251 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/81204eec1a8c42b5aa0fafb647e0f251 2024-11-16T10:16:12,171 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/467a14773d09419493e27406639e0418 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/467a14773d09419493e27406639e0418 2024-11-16T10:16:12,172 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/824284cee08c4ef4b6f77a899cf5a0e6 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/824284cee08c4ef4b6f77a899cf5a0e6 2024-11-16T10:16:12,173 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/5fd08f875ae44a22a2aaa57d19217765 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/5fd08f875ae44a22a2aaa57d19217765 2024-11-16T10:16:12,174 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/c3b3662725fb44a2ac4c85eacdb691c9 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/c3b3662725fb44a2ac4c85eacdb691c9 2024-11-16T10:16:12,176 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/fb7652a8350d472ca172af895fe0047f to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/fb7652a8350d472ca172af895fe0047f 2024-11-16T10:16:12,177 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/6024a0b701d441449e532f5757417f30 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/6024a0b701d441449e532f5757417f30 2024-11-16T10:16:12,181 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/12e5403801704924b5792f46c6714808, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/106301c006ce453e9b20a62341abb688, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/b7d807a6c6f8445dbeb657556861d3a3, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/aec028b42e1b47c89d2e480d686a7727, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/0a287c1205c543488d9c8ac0e739aa72, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/a14597648fc7444a879bf7d078556a46, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/7ac61e6e2b664427a19a04f577224723, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/b3b18aadd01f480687211cf8085d5999, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/37fa3907eefb4d2a8e26a8970c38f0c0, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/15fee025f79847dc8731d3272035e745, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/d2f0fb80ecea405e9ebbca846cab8653, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/fe50b08106424fb58e89b20a1cfc0d37, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/57b51273ebed4ac385c78587be1965ff, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/d25ce834cf544473863af106fdfcb965, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/af69d20769d146c0bc3a3038833c5784, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/d60d01f8a3994950ad0de5ab74ef2c83, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/b7e2b3a65a674427aac9216b6734a9de, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/193bfb818e9543d8aa7fab69cf9a8cba, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/3680cc212f864e74944237be52f26a82, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/20e76961b259474aa2f1da2ce9d7e780, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/999f9ad4b21a4b57a07b64ae7806d437, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/e005dc1626fb48d984adbe0cbab594bd, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/059283d1cb3a4f64b8dd59725d25377a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/643ed8d082c74da3b2e2e8a8893a19db, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/505dfa6435a540bea3cdf4ab9fd1940a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/0b1389423fe9406b9879390042f72b83] to archive 2024-11-16T10:16:12,182 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:16:12,184 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/12e5403801704924b5792f46c6714808 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/12e5403801704924b5792f46c6714808 2024-11-16T10:16:12,185 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/106301c006ce453e9b20a62341abb688 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/106301c006ce453e9b20a62341abb688 2024-11-16T10:16:12,186 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/b7d807a6c6f8445dbeb657556861d3a3 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/b7d807a6c6f8445dbeb657556861d3a3 2024-11-16T10:16:12,187 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/aec028b42e1b47c89d2e480d686a7727 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/aec028b42e1b47c89d2e480d686a7727 2024-11-16T10:16:12,188 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/0a287c1205c543488d9c8ac0e739aa72 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/0a287c1205c543488d9c8ac0e739aa72 2024-11-16T10:16:12,189 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/a14597648fc7444a879bf7d078556a46 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/a14597648fc7444a879bf7d078556a46 2024-11-16T10:16:12,190 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/7ac61e6e2b664427a19a04f577224723 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/7ac61e6e2b664427a19a04f577224723 2024-11-16T10:16:12,191 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/b3b18aadd01f480687211cf8085d5999 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/b3b18aadd01f480687211cf8085d5999 2024-11-16T10:16:12,192 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/37fa3907eefb4d2a8e26a8970c38f0c0 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/37fa3907eefb4d2a8e26a8970c38f0c0 2024-11-16T10:16:12,192 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/15fee025f79847dc8731d3272035e745 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/15fee025f79847dc8731d3272035e745 2024-11-16T10:16:12,193 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/d2f0fb80ecea405e9ebbca846cab8653 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/d2f0fb80ecea405e9ebbca846cab8653 2024-11-16T10:16:12,194 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/fe50b08106424fb58e89b20a1cfc0d37 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/fe50b08106424fb58e89b20a1cfc0d37 2024-11-16T10:16:12,195 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/57b51273ebed4ac385c78587be1965ff to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/57b51273ebed4ac385c78587be1965ff 2024-11-16T10:16:12,196 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/d25ce834cf544473863af106fdfcb965 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/d25ce834cf544473863af106fdfcb965 2024-11-16T10:16:12,196 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/af69d20769d146c0bc3a3038833c5784 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/af69d20769d146c0bc3a3038833c5784 2024-11-16T10:16:12,197 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/d60d01f8a3994950ad0de5ab74ef2c83 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/d60d01f8a3994950ad0de5ab74ef2c83 2024-11-16T10:16:12,197 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/b7e2b3a65a674427aac9216b6734a9de to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/b7e2b3a65a674427aac9216b6734a9de 2024-11-16T10:16:12,198 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/193bfb818e9543d8aa7fab69cf9a8cba to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/193bfb818e9543d8aa7fab69cf9a8cba 2024-11-16T10:16:12,199 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/3680cc212f864e74944237be52f26a82 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/3680cc212f864e74944237be52f26a82 2024-11-16T10:16:12,200 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/20e76961b259474aa2f1da2ce9d7e780 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/20e76961b259474aa2f1da2ce9d7e780 2024-11-16T10:16:12,200 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/999f9ad4b21a4b57a07b64ae7806d437 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/999f9ad4b21a4b57a07b64ae7806d437 2024-11-16T10:16:12,201 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/e005dc1626fb48d984adbe0cbab594bd to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/e005dc1626fb48d984adbe0cbab594bd 2024-11-16T10:16:12,202 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/059283d1cb3a4f64b8dd59725d25377a to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/059283d1cb3a4f64b8dd59725d25377a 2024-11-16T10:16:12,202 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/643ed8d082c74da3b2e2e8a8893a19db to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/643ed8d082c74da3b2e2e8a8893a19db 2024-11-16T10:16:12,203 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/505dfa6435a540bea3cdf4ab9fd1940a to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/505dfa6435a540bea3cdf4ab9fd1940a 2024-11-16T10:16:12,203 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/0b1389423fe9406b9879390042f72b83 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/0b1389423fe9406b9879390042f72b83 2024-11-16T10:16:13,959 INFO [Thread-1542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1542. 2024-11-16T10:16:13,959 DEBUG [Thread-1542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:16:13,959 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:13,959 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:13,959 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:16:13,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T10:16:13,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-16T10:16:13,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-16T10:16:13,961 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-16T10:16:13,961 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-16T10:16:13,961 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-16T10:16:13,961 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T10:16:13,961 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T10:16:13,961 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 968 2024-11-16T10:16:13,961 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2900 rows 2024-11-16T10:16:13,961 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 965 2024-11-16T10:16:13,961 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2891 rows 2024-11-16T10:16:13,961 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 958 2024-11-16T10:16:13,961 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2873 rows 2024-11-16T10:16:13,961 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 963 2024-11-16T10:16:13,961 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2885 rows 2024-11-16T10:16:13,961 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 960 2024-11-16T10:16:13,961 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2873 rows 2024-11-16T10:16:13,961 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T10:16:13,961 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T10:16:13,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:13,962 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:13,962 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:16:13,963 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T10:16:13,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T10:16:13,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T10:16:13,968 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752173968"}]},"ts":"1731752173968"} 2024-11-16T10:16:13,970 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T10:16:13,970 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T10:16:13,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T10:16:13,973 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6534162d1894b79648a1282b0163350b, UNASSIGN}] 2024-11-16T10:16:13,974 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6534162d1894b79648a1282b0163350b, UNASSIGN 2024-11-16T10:16:13,975 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=6534162d1894b79648a1282b0163350b, regionState=CLOSING, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:13,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6534162d1894b79648a1282b0163350b, UNASSIGN because future has completed 2024-11-16T10:16:13,976 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T10:16:13,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912}] 2024-11-16T10:16:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T10:16:14,129 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close 6534162d1894b79648a1282b0163350b 2024-11-16T10:16:14,129 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T10:16:14,129 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing 6534162d1894b79648a1282b0163350b, disabling compactions & flushes 2024-11-16T10:16:14,129 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:14,129 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:14,129 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. after waiting 0 ms 2024-11-16T10:16:14,129 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:14,129 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing 6534162d1894b79648a1282b0163350b 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-16T10:16:14,130 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=A 2024-11-16T10:16:14,130 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:14,130 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=B 2024-11-16T10:16:14,130 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:14,130 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6534162d1894b79648a1282b0163350b, store=C 2024-11-16T10:16:14,130 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:14,136 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c6fd0b21cc264a4092a63e302f05af98_6534162d1894b79648a1282b0163350b is 50, key is test_row_0/A:col10/1731752167875/Put/seqid=0 2024-11-16T10:16:14,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742271_1447 (size=9914) 2024-11-16T10:16:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T10:16:14,541 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:14,550 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c6fd0b21cc264a4092a63e302f05af98_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c6fd0b21cc264a4092a63e302f05af98_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:14,551 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/fea2128f2652417da0eca77cbc9d0a52, store: [table=TestAcidGuarantees family=A region=6534162d1894b79648a1282b0163350b] 2024-11-16T10:16:14,552 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/fea2128f2652417da0eca77cbc9d0a52 is 175, key is test_row_0/A:col10/1731752167875/Put/seqid=0 2024-11-16T10:16:14,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742272_1448 (size=22561) 2024-11-16T10:16:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T10:16:14,958 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=531, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/fea2128f2652417da0eca77cbc9d0a52 2024-11-16T10:16:14,973 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/72af646ddd404485b7d6a030e725eec6 is 50, key is test_row_0/B:col10/1731752167875/Put/seqid=0 2024-11-16T10:16:14,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742273_1449 (size=9857) 2024-11-16T10:16:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T10:16:15,377 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/72af646ddd404485b7d6a030e725eec6 2024-11-16T10:16:15,389 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/0580671b0d4f43a998cbf90096c7942d is 50, key is test_row_0/C:col10/1731752167875/Put/seqid=0 2024-11-16T10:16:15,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742274_1450 (size=9857) 2024-11-16T10:16:15,795 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/0580671b0d4f43a998cbf90096c7942d 2024-11-16T10:16:15,805 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/A/fea2128f2652417da0eca77cbc9d0a52 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/fea2128f2652417da0eca77cbc9d0a52 2024-11-16T10:16:15,809 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/fea2128f2652417da0eca77cbc9d0a52, entries=100, sequenceid=531, filesize=22.0 K 2024-11-16T10:16:15,810 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/B/72af646ddd404485b7d6a030e725eec6 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/72af646ddd404485b7d6a030e725eec6 2024-11-16T10:16:15,814 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/72af646ddd404485b7d6a030e725eec6, entries=100, sequenceid=531, filesize=9.6 K 2024-11-16T10:16:15,814 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/.tmp/C/0580671b0d4f43a998cbf90096c7942d as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/0580671b0d4f43a998cbf90096c7942d 2024-11-16T10:16:15,819 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/0580671b0d4f43a998cbf90096c7942d, entries=100, sequenceid=531, filesize=9.6 K 2024-11-16T10:16:15,820 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 6534162d1894b79648a1282b0163350b in 1690ms, sequenceid=531, compaction requested=false 2024-11-16T10:16:15,823 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/recovered.edits/534.seqid, newMaxSeqId=534, maxSeqId=4 2024-11-16T10:16:15,824 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. 2024-11-16T10:16:15,824 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for 6534162d1894b79648a1282b0163350b: Waiting for close lock at 1731752174129Running coprocessor pre-close hooks at 1731752174129Disabling compacts and flushes for region at 1731752174129Disabling writes for close at 1731752174129Obtaining lock to block concurrent updates at 1731752174129Preparing flush snapshotting stores in 6534162d1894b79648a1282b0163350b at 1731752174129Finished memstore snapshotting TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731752174130 (+1 ms)Flushing stores of TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b. at 1731752174130Flushing 6534162d1894b79648a1282b0163350b/A: creating writer at 1731752174130Flushing mob file 6534162d1894b79648a1282b0163350b/A: appending metadata at 1731752174136 (+6 ms)Flushing mob file 6534162d1894b79648a1282b0163350b/A: closing flushed file at 1731752174136Flushing 6534162d1894b79648a1282b0163350b/A: appending metadata at 1731752174551 (+415 ms)Flushing 6534162d1894b79648a1282b0163350b/A: closing flushed file at 1731752174551Flushing 6534162d1894b79648a1282b0163350b/B: creating writer at 1731752174967 (+416 ms)Flushing 6534162d1894b79648a1282b0163350b/B: appending metadata at 1731752174972 (+5 ms)Flushing 6534162d1894b79648a1282b0163350b/B: closing flushed file at 1731752174972Flushing 6534162d1894b79648a1282b0163350b/C: creating writer at 1731752175386 (+414 ms)Flushing 6534162d1894b79648a1282b0163350b/C: appending metadata at 1731752175389 (+3 ms)Flushing 6534162d1894b79648a1282b0163350b/C: closing flushed file at 1731752175389Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@544e7aed: reopening flushed file at 1731752175804 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43035f80: reopening flushed file at 1731752175809 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d87d7c6: reopening flushed file at 1731752175814 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 6534162d1894b79648a1282b0163350b in 1690ms, sequenceid=531, compaction requested=false at 1731752175820 (+6 ms)Writing region close event to WAL at 1731752175821 (+1 ms)Running coprocessor post-close hooks at 1731752175824 (+3 ms)Closed at 1731752175824 2024-11-16T10:16:15,825 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed 6534162d1894b79648a1282b0163350b 2024-11-16T10:16:15,826 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=6534162d1894b79648a1282b0163350b, regionState=CLOSED 2024-11-16T10:16:15,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 because future has completed 2024-11-16T10:16:15,829 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-16T10:16:15,829 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure 6534162d1894b79648a1282b0163350b, server=b16ff38458c1,45417,1731752048912 in 1.8510 sec 2024-11-16T10:16:15,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-16T10:16:15,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6534162d1894b79648a1282b0163350b, UNASSIGN in 1.8560 sec 2024-11-16T10:16:15,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-16T10:16:15,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8600 sec 2024-11-16T10:16:15,833 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752175833"}]},"ts":"1731752175833"} 2024-11-16T10:16:15,834 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T10:16:15,834 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T10:16:15,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8710 sec 2024-11-16T10:16:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T10:16:16,116 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:16,117 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T10:16:16,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:16:16,122 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:16:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T10:16:16,124 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:16:16,128 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b 2024-11-16T10:16:16,131 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A, FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B, FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C, FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/recovered.edits] 2024-11-16T10:16:16,134 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/aec6de8b590b4b498e7beeaeaec4718a to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/aec6de8b590b4b498e7beeaeaec4718a 2024-11-16T10:16:16,136 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/fea2128f2652417da0eca77cbc9d0a52 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/A/fea2128f2652417da0eca77cbc9d0a52 2024-11-16T10:16:16,140 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/11b4da8d4b4a4572969ec5fa17c78e79 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/11b4da8d4b4a4572969ec5fa17c78e79 2024-11-16T10:16:16,141 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/72af646ddd404485b7d6a030e725eec6 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/B/72af646ddd404485b7d6a030e725eec6 2024-11-16T10:16:16,145 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/0580671b0d4f43a998cbf90096c7942d to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/0580671b0d4f43a998cbf90096c7942d 2024-11-16T10:16:16,147 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/b44c8aa7db7e4e858bba0d5f023f9df3 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/C/b44c8aa7db7e4e858bba0d5f023f9df3 2024-11-16T10:16:16,152 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/recovered.edits/534.seqid to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b/recovered.edits/534.seqid 2024-11-16T10:16:16,153 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/6534162d1894b79648a1282b0163350b 2024-11-16T10:16:16,153 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T10:16:16,154 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T10:16:16,155 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T10:16:16,161 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116152a7168b402421f81871c5bbae70a1e_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116152a7168b402421f81871c5bbae70a1e_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:16,163 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116219057d193c442f99f0f48b0dbff2476_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116219057d193c442f99f0f48b0dbff2476_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:16,166 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111646cb0fce803d467688f3b3ff22e1b322_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111646cb0fce803d467688f3b3ff22e1b322_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:16,168 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111668f5d750acdf4d3ab81c5f8d7c0416d4_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111668f5d750acdf4d3ab81c5f8d7c0416d4_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:16,170 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111670cf7898f5cc4da289ba55d0f6edacfd_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111670cf7898f5cc4da289ba55d0f6edacfd_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:16,172 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167497f3bb3b1945a0afbc226ef39fad5f_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167497f3bb3b1945a0afbc226ef39fad5f_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:16,174 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167c5127d515f84ec3af73e71d02a52489_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167c5127d515f84ec3af73e71d02a52489_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:16,175 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167ce7d4ca7c184079abac0c8e504e3fe2_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167ce7d4ca7c184079abac0c8e504e3fe2_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:16,177 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b79c76e8cd51450b924c22767232fbd1_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b79c76e8cd51450b924c22767232fbd1_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:16,178 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c6686b491d6c45e0b739db43d2f407d4_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c6686b491d6c45e0b739db43d2f407d4_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:16,180 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c6fd0b21cc264a4092a63e302f05af98_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c6fd0b21cc264a4092a63e302f05af98_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:16,181 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c8f040857e8444b88fe1697f03529c7f_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c8f040857e8444b88fe1697f03529c7f_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:16,182 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c910071cc1a044e48038aee25a68a63b_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c910071cc1a044e48038aee25a68a63b_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:16,184 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c99c386473e246a696c296fca0cf36d5_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c99c386473e246a696c296fca0cf36d5_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:16,185 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116cf9b7f53feb54a9f86dc0fa9f21d305a_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116cf9b7f53feb54a9f86dc0fa9f21d305a_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:16,186 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d994fd3c9ad348089341bfceba3b0ab9_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d994fd3c9ad348089341bfceba3b0ab9_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:16,188 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e449814f90914fad8000b34ebe08fbb8_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e449814f90914fad8000b34ebe08fbb8_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:16,189 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f0df4f1f16134c4abfee51794df6a36d_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f0df4f1f16134c4abfee51794df6a36d_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:16,191 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f1caeb907246498ca89cd5c06026e209_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f1caeb907246498ca89cd5c06026e209_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:16,192 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fc157365295d4260b9b2aec30f94e2b4_6534162d1894b79648a1282b0163350b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fc157365295d4260b9b2aec30f94e2b4_6534162d1894b79648a1282b0163350b 2024-11-16T10:16:16,192 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T10:16:16,194 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:16:16,196 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T10:16:16,197 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T10:16:16,198 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:16:16,198 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T10:16:16,198 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731752176198"}]},"ts":"9223372036854775807"} 2024-11-16T10:16:16,199 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T10:16:16,200 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6534162d1894b79648a1282b0163350b, NAME => 'TestAcidGuarantees,,1731752144211.6534162d1894b79648a1282b0163350b.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T10:16:16,200 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T10:16:16,200 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731752176200"}]},"ts":"9223372036854775807"} 2024-11-16T10:16:16,201 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T10:16:16,201 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:16:16,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 84 msec 2024-11-16T10:16:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T10:16:16,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T10:16:16,234 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:16,246 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 248), OpenFileDescriptor=449 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=402 (was 406), ProcessCount=11 (was 11), AvailableMemoryMB=3690 (was 4212) 2024-11-16T10:16:16,256 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=402, ProcessCount=11, AvailableMemoryMB=3690 2024-11-16T10:16:16,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T10:16:16,257 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T10:16:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T10:16:16,259 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T10:16:16,259 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:16,259 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-11-16T10:16:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-16T10:16:16,259 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T10:16:16,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742275_1451 (size=1061) 2024-11-16T10:16:16,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-16T10:16:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-16T10:16:16,666 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0 2024-11-16T10:16:16,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742276_1452 (size=53) 2024-11-16T10:16:16,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-16T10:16:17,071 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:16:17,071 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 77e3e5908c8d586f7cad974ff9b02535, disabling compactions & flushes 2024-11-16T10:16:17,071 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:17,071 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:17,071 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. after waiting 0 ms 2024-11-16T10:16:17,071 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:17,071 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:17,071 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 77e3e5908c8d586f7cad974ff9b02535: Waiting for close lock at 1731752177071Disabling compacts and flushes for region at 1731752177071Disabling writes for close at 1731752177071Writing region close event to WAL at 1731752177071Closed at 1731752177071 2024-11-16T10:16:17,114 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T10:16:17,115 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731752177115"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731752177115"}]},"ts":"1731752177115"} 2024-11-16T10:16:17,119 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T10:16:17,120 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T10:16:17,121 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752177120"}]},"ts":"1731752177120"} 2024-11-16T10:16:17,123 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T10:16:17,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77e3e5908c8d586f7cad974ff9b02535, ASSIGN}] 2024-11-16T10:16:17,126 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77e3e5908c8d586f7cad974ff9b02535, ASSIGN 2024-11-16T10:16:17,127 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=77e3e5908c8d586f7cad974ff9b02535, ASSIGN; state=OFFLINE, location=b16ff38458c1,45417,1731752048912; forceNewPlan=false, retain=false 2024-11-16T10:16:17,278 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=77e3e5908c8d586f7cad974ff9b02535, regionState=OPENING, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:17,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=77e3e5908c8d586f7cad974ff9b02535, ASSIGN because future has completed 2024-11-16T10:16:17,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912}] 2024-11-16T10:16:17,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-16T10:16:17,440 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:17,441 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:16:17,442 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:17,443 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:16:17,443 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:17,443 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:17,446 INFO [StoreOpener-77e3e5908c8d586f7cad974ff9b02535-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:17,447 INFO [StoreOpener-77e3e5908c8d586f7cad974ff9b02535-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:16:17,447 INFO [StoreOpener-77e3e5908c8d586f7cad974ff9b02535-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 77e3e5908c8d586f7cad974ff9b02535 columnFamilyName A 2024-11-16T10:16:17,447 DEBUG [StoreOpener-77e3e5908c8d586f7cad974ff9b02535-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:17,448 INFO [StoreOpener-77e3e5908c8d586f7cad974ff9b02535-1 {}] regionserver.HStore(327): Store=77e3e5908c8d586f7cad974ff9b02535/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:16:17,448 INFO [StoreOpener-77e3e5908c8d586f7cad974ff9b02535-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:17,449 INFO [StoreOpener-77e3e5908c8d586f7cad974ff9b02535-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:16:17,449 INFO [StoreOpener-77e3e5908c8d586f7cad974ff9b02535-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 77e3e5908c8d586f7cad974ff9b02535 columnFamilyName B 2024-11-16T10:16:17,449 DEBUG [StoreOpener-77e3e5908c8d586f7cad974ff9b02535-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:17,449 INFO [StoreOpener-77e3e5908c8d586f7cad974ff9b02535-1 {}] regionserver.HStore(327): Store=77e3e5908c8d586f7cad974ff9b02535/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:16:17,449 INFO [StoreOpener-77e3e5908c8d586f7cad974ff9b02535-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:17,450 INFO [StoreOpener-77e3e5908c8d586f7cad974ff9b02535-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:16:17,451 INFO [StoreOpener-77e3e5908c8d586f7cad974ff9b02535-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 77e3e5908c8d586f7cad974ff9b02535 columnFamilyName C 2024-11-16T10:16:17,451 DEBUG [StoreOpener-77e3e5908c8d586f7cad974ff9b02535-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:17,451 INFO [StoreOpener-77e3e5908c8d586f7cad974ff9b02535-1 {}] regionserver.HStore(327): Store=77e3e5908c8d586f7cad974ff9b02535/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:16:17,451 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:17,451 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:17,452 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:17,452 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:17,453 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:17,453 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:17,454 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:16:17,455 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:17,457 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T10:16:17,457 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened 77e3e5908c8d586f7cad974ff9b02535; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62038002, jitterRate=-0.07556173205375671}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:16:17,457 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:17,458 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for 77e3e5908c8d586f7cad974ff9b02535: Running coprocessor pre-open hook at 1731752177443Writing region info on filesystem at 1731752177443Initializing all the Stores at 1731752177445 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752177445Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752177445Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752177445Cleaning up temporary data from old regions at 1731752177453 (+8 ms)Running coprocessor post-open hooks at 1731752177457 (+4 ms)Region opened successfully at 1731752177458 (+1 ms) 2024-11-16T10:16:17,459 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., pid=120, masterSystemTime=1731752177435 2024-11-16T10:16:17,461 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:17,461 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:17,462 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=77e3e5908c8d586f7cad974ff9b02535, regionState=OPEN, openSeqNum=2, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:17,463 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 because future has completed 2024-11-16T10:16:17,466 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-16T10:16:17,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure 77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 in 182 msec 2024-11-16T10:16:17,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-16T10:16:17,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77e3e5908c8d586f7cad974ff9b02535, ASSIGN in 343 msec 2024-11-16T10:16:17,470 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T10:16:17,470 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752177470"}]},"ts":"1731752177470"} 2024-11-16T10:16:17,472 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T10:16:17,473 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T10:16:17,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2160 sec 2024-11-16T10:16:18,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-16T10:16:18,405 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:18,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be1708d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,407 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:16:18,408 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:16:18,409 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:16:18,409 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:16:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:16:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c805bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:16:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:16:18,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:18,412 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:16:18,413 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d424e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,413 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:16:18,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:16:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:16:18,416 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:16:18,417 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T10:16:18,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is b16ff38458c1,45949,1731752047888 2024-11-16T10:16:18,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@f8209db 2024-11-16T10:16:18,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T10:16:18,420 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T10:16:18,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fe0696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,422 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:16:18,422 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:16:18,423 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:16:18,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:16:18,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:16:18,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59189f63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:16:18,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:16:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:18,424 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:16:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2333ca71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59de117f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,426 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:16:18,426 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:16:18,427 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:16:18,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:16:18,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:16:18,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@756be490, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:16:18,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:16:18,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:18,429 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:16:18,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@576b49ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472a8168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,430 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:16:18,431 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:16:18,431 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:16:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:16:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:16:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@254d6c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:16:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:16:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:18,433 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:16:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a24e43a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,434 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f010827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,434 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:16:18,434 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:16:18,435 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:16:18,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:16:18,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:16:18,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d6bc8c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:16:18,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:16:18,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:18,436 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:16:18,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74fc8d29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2822c33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,438 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:16:18,438 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:16:18,439 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:16:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:16:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:16:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cd2aa81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:16:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:16:18,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:18,440 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:16:18,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e516664, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc5e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,442 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:16:18,442 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:16:18,443 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:16:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:16:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:16:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f4daf49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:16:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:16:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:18,444 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:16:18,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c6d3ab5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42daa6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,446 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:16:18,446 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:16:18,447 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:16:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:16:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:16:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@289be103, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:16:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:16:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:18,448 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:16:18,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39547fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366fb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,449 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:16:18,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:16:18,449 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:16:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:16:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:16:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fdabe61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:16:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:16:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:18,451 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:16:18,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@495263f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,451 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e8e518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,451 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:16:18,452 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:16:18,452 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:16:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:16:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:16:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ceb169d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:16:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:16:18,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:18,453 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:16:18,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1475e648, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,454 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ff7be5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,454 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:16:18,454 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:16:18,455 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:16:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:16:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:16:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c8f12d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:16:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:16:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:18,456 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:16:18,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c020f7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:18,456 DEBUG [Thread-2030 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:16:18,457 DEBUG [Thread-2028 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:16:18,457 DEBUG [Thread-2045 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:16:18,457 DEBUG [Thread-2037 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:16:18,457 DEBUG [Thread-2043 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:16:18,457 DEBUG [Thread-2034 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:16:18,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:16:18,457 DEBUG [Thread-2032 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:16:18,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:16:18,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:16:18,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:16:18,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:16:18,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:16:18,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:16:18,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:16:18,458 DEBUG [Thread-2041 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:16:18,458 DEBUG [Thread-2039 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:16:18,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:16:18,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:16:18,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:16:18,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:16:18,459 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:16:18,459 DEBUG [Thread-2026 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:16:18,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:16:18,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:16:18,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:16:18,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:16:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-16T10:16:18,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:16:18,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:16:18,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:16:18,460 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:16:18,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:16:18,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:16:18,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:16:18,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:16:18,461 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:16:18,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:16:18,461 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:16:18,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:16:18,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:16:18,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T10:16:18,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:16:18,462 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:16:18,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:16:18,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:16:18,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:16:18,463 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:16:18,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:16:18,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:16:18,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:16:18,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:16:18,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:16:18,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:16:18,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:16:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2] 2024-11-16T10:16:18,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:18,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77e3e5908c8d586f7cad974ff9b02535 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T10:16:18,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=A 2024-11-16T10:16:18,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:18,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=B 2024-11-16T10:16:18,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:18,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=C 2024-11-16T10:16:18,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:18,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/a962b08f2960446b96029c08580cc78c is 50, key is test_row_0/A:col10/1731752178466/Put/seqid=0 2024-11-16T10:16:18,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742277_1453 (size=16681) 2024-11-16T10:16:18,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/a962b08f2960446b96029c08580cc78c 2024-11-16T10:16:18,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731752178492, max timestamp=1731752178508Num uniques 859; 2024-11-16T10:16:18,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731752178492, max timestamp=1731752178508Num uniques 859; 2024-11-16T10:16:18,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T10:16:18,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T10:16:18,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731752178492, max timestamp=1731752178508Num uniques 859; 2024-11-16T10:16:18,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T10:16:18,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/3da868d564fc46f582b12eac759a290f is 50, key is test_row_0/B:col10/1731752178466/Put/seqid=0 2024-11-16T10:16:18,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742278_1454 (size=12001) 2024-11-16T10:16:18,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752238534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:18,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752238539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:18,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:18,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:18,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752238540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:18,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:18,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51766 deadline: 1731752238547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:18,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:18,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:18,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1731752238549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T10:16:18,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T10:16:18,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:18,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:18,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:18,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:18,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:18,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:18,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:18,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752238654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:18,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752238654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:18,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:18,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752238664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:18,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:18,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51766 deadline: 1731752238664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:18,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:18,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1731752238674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:18,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:18,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T10:16:18,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:18,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:18,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:18,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:18,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T10:16:18,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752238864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:18,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:18,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752238874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:18,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:18,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752238884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:18,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:18,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51766 deadline: 1731752238884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:18,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:18,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:18,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1731752238894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:18,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T10:16:18,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:18,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:18,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:18,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:18,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:18,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/3da868d564fc46f582b12eac759a290f 2024-11-16T10:16:18,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/6716fc154e024be489d03e0d7fb83114 is 50, key is test_row_0/C:col10/1731752178466/Put/seqid=0 2024-11-16T10:16:18,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742279_1455 (size=12001) 2024-11-16T10:16:19,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T10:16:19,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:19,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:19,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:19,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:19,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T10:16:19,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752239174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:19,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:19,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752239193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:19,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:19,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752239204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:19,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:19,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:19,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51766 deadline: 1731752239204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:19,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:19,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:19,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1731752239204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:19,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:19,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T10:16:19,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:19,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:19,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:19,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:19,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:19,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:19,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/6716fc154e024be489d03e0d7fb83114 2024-11-16T10:16:19,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/a962b08f2960446b96029c08580cc78c as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/a962b08f2960446b96029c08580cc78c 2024-11-16T10:16:19,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/a962b08f2960446b96029c08580cc78c, entries=250, sequenceid=11, filesize=16.3 K 2024-11-16T10:16:19,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/3da868d564fc46f582b12eac759a290f as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/3da868d564fc46f582b12eac759a290f 2024-11-16T10:16:19,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/3da868d564fc46f582b12eac759a290f, entries=150, sequenceid=11, filesize=11.7 K 2024-11-16T10:16:19,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/6716fc154e024be489d03e0d7fb83114 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/6716fc154e024be489d03e0d7fb83114 2024-11-16T10:16:19,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T10:16:19,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:19,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:19,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:19,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:19,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:19,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/6716fc154e024be489d03e0d7fb83114, entries=150, sequenceid=11, filesize=11.7 K 2024-11-16T10:16:19,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 77e3e5908c8d586f7cad974ff9b02535 in 915ms, sequenceid=11, compaction requested=false 2024-11-16T10:16:19,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:19,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T10:16:19,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:19,534 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 77e3e5908c8d586f7cad974ff9b02535 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-16T10:16:19,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=A 2024-11-16T10:16:19,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:19,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=B 2024-11-16T10:16:19,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:19,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=C 2024-11-16T10:16:19,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:19,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/0b9325598db840039fbd396828c111b0 is 50, key is test_row_0/A:col10/1731752178539/Put/seqid=0 2024-11-16T10:16:19,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742280_1456 (size=12001) 2024-11-16T10:16:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T10:16:19,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:19,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:19,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752239719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:19,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51766 deadline: 1731752239724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:19,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1731752239724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:19,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:19,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752239743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:19,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:19,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752239743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:19,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:19,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:19,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:19,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752239854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:19,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:19,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752239864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:19,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:19,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752239864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:19,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:19,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:19,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:19,942 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/0b9325598db840039fbd396828c111b0 2024-11-16T10:16:19,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/a035f46ac57d4f658ef1b662998ecc84 is 50, key is test_row_0/B:col10/1731752178539/Put/seqid=0 2024-11-16T10:16:19,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742281_1457 (size=12001) 2024-11-16T10:16:19,957 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/a035f46ac57d4f658ef1b662998ecc84 2024-11-16T10:16:19,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/c205c7501bd94a8081a6be8d8cb0dfd4 is 50, key is test_row_0/C:col10/1731752178539/Put/seqid=0 2024-11-16T10:16:19,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742282_1458 (size=12001) 2024-11-16T10:16:20,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:20,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752240074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:20,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752240074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:20,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:20,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:20,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752240074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:20,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:20,374 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/c205c7501bd94a8081a6be8d8cb0dfd4 2024-11-16T10:16:20,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/0b9325598db840039fbd396828c111b0 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/0b9325598db840039fbd396828c111b0 2024-11-16T10:16:20,380 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/0b9325598db840039fbd396828c111b0, entries=150, sequenceid=46, filesize=11.7 K 2024-11-16T10:16:20,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/a035f46ac57d4f658ef1b662998ecc84 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/a035f46ac57d4f658ef1b662998ecc84 2024-11-16T10:16:20,384 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/a035f46ac57d4f658ef1b662998ecc84, entries=150, sequenceid=46, filesize=11.7 K 2024-11-16T10:16:20,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/c205c7501bd94a8081a6be8d8cb0dfd4 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/c205c7501bd94a8081a6be8d8cb0dfd4 2024-11-16T10:16:20,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752240384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:20,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:20,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752240384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:20,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752240384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:20,391 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/c205c7501bd94a8081a6be8d8cb0dfd4, entries=150, sequenceid=46, filesize=11.7 K 2024-11-16T10:16:20,392 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for 77e3e5908c8d586f7cad974ff9b02535 in 857ms, sequenceid=46, compaction requested=false 2024-11-16T10:16:20,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:20,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:20,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-16T10:16:20,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-16T10:16:20,394 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-16T10:16:20,394 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9300 sec 2024-11-16T10:16:20,396 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.9360 sec 2024-11-16T10:16:20,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T10:16:20,614 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:20,615 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:16:20,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-16T10:16:20,617 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:16:20,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T10:16:20,618 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:16:20,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:16:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T10:16:20,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:20,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77e3e5908c8d586f7cad974ff9b02535 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T10:16:20,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=A 2024-11-16T10:16:20,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:20,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=B 2024-11-16T10:16:20,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:20,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=C 2024-11-16T10:16:20,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:20,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/28bb02a02d6a42f3b73601b5eb7b408f is 50, key is test_row_0/A:col10/1731752179731/Put/seqid=0 2024-11-16T10:16:20,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742283_1459 (size=14341) 2024-11-16T10:16:20,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T10:16:20,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:20,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:20,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:20,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:20,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:20,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1731752240829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:20,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:20,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51766 deadline: 1731752240830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:20,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:20,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752240894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:20,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752240894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752240894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:20,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T10:16:20,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:20,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:20,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:20,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:20,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T10:16:20,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1731752240944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:20,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:20,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51766 deadline: 1731752240954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:21,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T10:16:21,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:21,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:21,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:21,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:21,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:21,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/28bb02a02d6a42f3b73601b5eb7b408f 2024-11-16T10:16:21,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:21,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1731752241164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:21,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/2376d5e22c9f48ddb0f6749adf728ec8 is 50, key is test_row_0/B:col10/1731752179731/Put/seqid=0 2024-11-16T10:16:21,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742284_1460 (size=12001) 2024-11-16T10:16:21,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51766 deadline: 1731752241173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:21,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T10:16:21,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:21,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:21,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:21,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:21,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T10:16:21,345 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T10:16:21,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T10:16:21,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:21,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:21,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:21,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:21,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:21,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:21,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1731752241474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:21,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:21,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51766 deadline: 1731752241494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:21,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:21,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:21,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:21,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T10:16:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:21,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:21,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/2376d5e22c9f48ddb0f6749adf728ec8 2024-11-16T10:16:21,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/240c2f3040664c1c8d9b8c16f9d22d83 is 50, key is test_row_0/C:col10/1731752179731/Put/seqid=0 2024-11-16T10:16:21,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742285_1461 (size=12001) 2024-11-16T10:16:21,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/240c2f3040664c1c8d9b8c16f9d22d83 2024-11-16T10:16:21,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/28bb02a02d6a42f3b73601b5eb7b408f as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/28bb02a02d6a42f3b73601b5eb7b408f 2024-11-16T10:16:21,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/28bb02a02d6a42f3b73601b5eb7b408f, entries=200, sequenceid=58, filesize=14.0 K 2024-11-16T10:16:21,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/2376d5e22c9f48ddb0f6749adf728ec8 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/2376d5e22c9f48ddb0f6749adf728ec8 2024-11-16T10:16:21,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/2376d5e22c9f48ddb0f6749adf728ec8, entries=150, sequenceid=58, filesize=11.7 K 2024-11-16T10:16:21,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/240c2f3040664c1c8d9b8c16f9d22d83 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/240c2f3040664c1c8d9b8c16f9d22d83 2024-11-16T10:16:21,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/240c2f3040664c1c8d9b8c16f9d22d83, entries=150, sequenceid=58, filesize=11.7 K 2024-11-16T10:16:21,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 77e3e5908c8d586f7cad974ff9b02535 in 893ms, sequenceid=58, compaction requested=true 2024-11-16T10:16:21,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:21,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77e3e5908c8d586f7cad974ff9b02535:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:16:21,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:21,652 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:21,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77e3e5908c8d586f7cad974ff9b02535:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:16:21,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:21,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77e3e5908c8d586f7cad974ff9b02535:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:16:21,652 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:21,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:21,653 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:21,653 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 77e3e5908c8d586f7cad974ff9b02535/B is initiating minor compaction (all files) 2024-11-16T10:16:21,653 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:21,653 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 77e3e5908c8d586f7cad974ff9b02535/A is initiating minor compaction (all files) 2024-11-16T10:16:21,653 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77e3e5908c8d586f7cad974ff9b02535/B in TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:21,653 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77e3e5908c8d586f7cad974ff9b02535/A in TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:21,653 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/a962b08f2960446b96029c08580cc78c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/0b9325598db840039fbd396828c111b0, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/28bb02a02d6a42f3b73601b5eb7b408f] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp, totalSize=42.0 K 2024-11-16T10:16:21,653 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/3da868d564fc46f582b12eac759a290f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/a035f46ac57d4f658ef1b662998ecc84, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/2376d5e22c9f48ddb0f6749adf728ec8] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp, totalSize=35.2 K 2024-11-16T10:16:21,654 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 3da868d564fc46f582b12eac759a290f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731752178466 2024-11-16T10:16:21,654 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting a962b08f2960446b96029c08580cc78c, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731752178464 2024-11-16T10:16:21,654 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b9325598db840039fbd396828c111b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731752178529 2024-11-16T10:16:21,654 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting a035f46ac57d4f658ef1b662998ecc84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731752178529 2024-11-16T10:16:21,654 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28bb02a02d6a42f3b73601b5eb7b408f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731752179704 2024-11-16T10:16:21,654 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 2376d5e22c9f48ddb0f6749adf728ec8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731752179719 2024-11-16T10:16:21,670 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77e3e5908c8d586f7cad974ff9b02535#B#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:21,671 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/c32445e4762a4bb5a6a3fc991ac567ed is 50, key is test_row_0/B:col10/1731752179731/Put/seqid=0 2024-11-16T10:16:21,677 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77e3e5908c8d586f7cad974ff9b02535#A#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:21,679 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/f54f9d722a98451a96f5a9448d74b2ec is 50, key is test_row_0/A:col10/1731752179731/Put/seqid=0 2024-11-16T10:16:21,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742286_1462 (size=12104) 2024-11-16T10:16:21,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T10:16:21,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:21,687 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 77e3e5908c8d586f7cad974ff9b02535 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T10:16:21,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=A 2024-11-16T10:16:21,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:21,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=B 2024-11-16T10:16:21,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:21,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=C 2024-11-16T10:16:21,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:21,688 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/c32445e4762a4bb5a6a3fc991ac567ed as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/c32445e4762a4bb5a6a3fc991ac567ed 2024-11-16T10:16:21,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/2c831c8721424193b9098c21e1702d26 is 50, key is test_row_0/A:col10/1731752180815/Put/seqid=0 2024-11-16T10:16:21,697 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77e3e5908c8d586f7cad974ff9b02535/B of 77e3e5908c8d586f7cad974ff9b02535 into c32445e4762a4bb5a6a3fc991ac567ed(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:21,697 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:21,697 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., storeName=77e3e5908c8d586f7cad974ff9b02535/B, priority=13, startTime=1731752181652; duration=0sec 2024-11-16T10:16:21,697 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:21,697 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77e3e5908c8d586f7cad974ff9b02535:B 2024-11-16T10:16:21,697 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:21,698 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:21,698 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 77e3e5908c8d586f7cad974ff9b02535/C is initiating minor compaction (all files) 2024-11-16T10:16:21,698 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77e3e5908c8d586f7cad974ff9b02535/C in TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:21,699 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/6716fc154e024be489d03e0d7fb83114, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/c205c7501bd94a8081a6be8d8cb0dfd4, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/240c2f3040664c1c8d9b8c16f9d22d83] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp, totalSize=35.2 K 2024-11-16T10:16:21,699 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 6716fc154e024be489d03e0d7fb83114, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731752178466 2024-11-16T10:16:21,699 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting c205c7501bd94a8081a6be8d8cb0dfd4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731752178529 2024-11-16T10:16:21,700 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 240c2f3040664c1c8d9b8c16f9d22d83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731752179719 2024-11-16T10:16:21,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742287_1463 (size=12104) 2024-11-16T10:16:21,710 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/f54f9d722a98451a96f5a9448d74b2ec as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/f54f9d722a98451a96f5a9448d74b2ec 2024-11-16T10:16:21,716 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77e3e5908c8d586f7cad974ff9b02535/A of 77e3e5908c8d586f7cad974ff9b02535 into f54f9d722a98451a96f5a9448d74b2ec(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:21,716 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:21,716 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., storeName=77e3e5908c8d586f7cad974ff9b02535/A, priority=13, startTime=1731752181652; duration=0sec 2024-11-16T10:16:21,716 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:21,716 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77e3e5908c8d586f7cad974ff9b02535:A 2024-11-16T10:16:21,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742288_1464 (size=12001) 2024-11-16T10:16:21,722 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77e3e5908c8d586f7cad974ff9b02535#C#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:21,722 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/93ffddd471984f0cad08134b05f37500 is 50, key is test_row_0/C:col10/1731752179731/Put/seqid=0 2024-11-16T10:16:21,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742289_1465 (size=12104) 2024-11-16T10:16:21,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T10:16:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:21,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:21,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752241933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:21,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752241938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:21,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:21,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:21,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:21,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752241939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:22,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1731752241994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:22,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:22,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:22,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:22,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51766 deadline: 1731752242014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:22,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:22,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:22,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:22,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752242043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:22,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752242054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:22,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:22,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:22,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752242054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:22,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:22,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:22,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:22,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:22,118 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/2c831c8721424193b9098c21e1702d26 2024-11-16T10:16:22,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/e5c1918b1e66402181b601e1158bbe88 is 50, key is test_row_0/B:col10/1731752180815/Put/seqid=0 2024-11-16T10:16:22,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742290_1466 (size=12001) 2024-11-16T10:16:22,131 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/93ffddd471984f0cad08134b05f37500 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/93ffddd471984f0cad08134b05f37500 2024-11-16T10:16:22,135 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77e3e5908c8d586f7cad974ff9b02535/C of 77e3e5908c8d586f7cad974ff9b02535 into 93ffddd471984f0cad08134b05f37500(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:22,135 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:22,135 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., storeName=77e3e5908c8d586f7cad974ff9b02535/C, priority=13, startTime=1731752181652; duration=0sec 2024-11-16T10:16:22,135 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:22,135 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77e3e5908c8d586f7cad974ff9b02535:C 2024-11-16T10:16:22,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752242263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:22,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:22,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:22,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:22,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752242274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:22,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:22,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:22,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:22,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752242274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:22,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:22,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:22,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:22,527 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/e5c1918b1e66402181b601e1158bbe88 2024-11-16T10:16:22,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/f7e294aa632a4d8b8cf313fe977d1112 is 50, key is test_row_0/C:col10/1731752180815/Put/seqid=0 2024-11-16T10:16:22,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742291_1467 (size=12001) 2024-11-16T10:16:22,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752242574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:22,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:22,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:22,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:22,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752242594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:22,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752242594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T10:16:22,940 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/f7e294aa632a4d8b8cf313fe977d1112 2024-11-16T10:16:22,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/2c831c8721424193b9098c21e1702d26 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/2c831c8721424193b9098c21e1702d26 2024-11-16T10:16:22,947 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/2c831c8721424193b9098c21e1702d26, entries=150, sequenceid=78, filesize=11.7 K 2024-11-16T10:16:22,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/e5c1918b1e66402181b601e1158bbe88 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/e5c1918b1e66402181b601e1158bbe88 2024-11-16T10:16:22,950 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/e5c1918b1e66402181b601e1158bbe88, entries=150, sequenceid=78, filesize=11.7 K 2024-11-16T10:16:22,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/f7e294aa632a4d8b8cf313fe977d1112 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/f7e294aa632a4d8b8cf313fe977d1112 2024-11-16T10:16:22,953 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/f7e294aa632a4d8b8cf313fe977d1112, entries=150, sequenceid=78, filesize=11.7 K 2024-11-16T10:16:22,954 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 77e3e5908c8d586f7cad974ff9b02535 in 1267ms, sequenceid=78, compaction requested=false 2024-11-16T10:16:22,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:22,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:22,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-16T10:16:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-16T10:16:22,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-16T10:16:22,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3380 sec 2024-11-16T10:16:22,959 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.3430 sec 2024-11-16T10:16:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:23,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77e3e5908c8d586f7cad974ff9b02535 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T10:16:23,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=A 2024-11-16T10:16:23,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:23,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=B 2024-11-16T10:16:23,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:23,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=C 2024-11-16T10:16:23,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:23,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/54435b39ec79438aad50d72494957363 is 50, key is test_row_0/A:col10/1731752183016/Put/seqid=0 2024-11-16T10:16:23,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742292_1468 (size=11997) 2024-11-16T10:16:23,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/54435b39ec79438aad50d72494957363 2024-11-16T10:16:23,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/28348b7200714fad87cad418c32ab69b is 50, key is test_row_0/B:col10/1731752183016/Put/seqid=0 2024-11-16T10:16:23,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742293_1469 (size=9657) 2024-11-16T10:16:23,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51766 deadline: 1731752243080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:23,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:23,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:23,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:23,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1731752243081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:23,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:23,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:23,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752243084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:23,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:23,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:23,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:23,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:23,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752243104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:23,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752243104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:23,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:23,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:23,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:23,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51766 deadline: 1731752243194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:23,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:23,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:23,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:23,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1731752243203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:23,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:23,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:23,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:23,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:23,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51766 deadline: 1731752243414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:23,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:23,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:23,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:23,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1731752243424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:23,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:23,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:23,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:23,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/28348b7200714fad87cad418c32ab69b 2024-11-16T10:16:23,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/5e04f8849bb24646a3cbb4080792e154 is 50, key is test_row_0/C:col10/1731752183016/Put/seqid=0 2024-11-16T10:16:23,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742294_1470 (size=9657) 2024-11-16T10:16:23,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51766 deadline: 1731752243724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:23,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:23,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:23,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:23,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1731752243733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:23,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:23,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:23,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:23,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/5e04f8849bb24646a3cbb4080792e154 2024-11-16T10:16:23,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/54435b39ec79438aad50d72494957363 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/54435b39ec79438aad50d72494957363 2024-11-16T10:16:23,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/54435b39ec79438aad50d72494957363, entries=150, sequenceid=94, filesize=11.7 K 2024-11-16T10:16:23,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/28348b7200714fad87cad418c32ab69b as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/28348b7200714fad87cad418c32ab69b 2024-11-16T10:16:23,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/28348b7200714fad87cad418c32ab69b, entries=100, sequenceid=94, filesize=9.4 K 2024-11-16T10:16:23,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/5e04f8849bb24646a3cbb4080792e154 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/5e04f8849bb24646a3cbb4080792e154 2024-11-16T10:16:23,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/5e04f8849bb24646a3cbb4080792e154, entries=100, sequenceid=94, filesize=9.4 K 2024-11-16T10:16:23,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 77e3e5908c8d586f7cad974ff9b02535 in 852ms, sequenceid=94, compaction requested=true 2024-11-16T10:16:23,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:23,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77e3e5908c8d586f7cad974ff9b02535:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:16:23,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:23,870 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:23,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77e3e5908c8d586f7cad974ff9b02535:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:16:23,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:23,870 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:23,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77e3e5908c8d586f7cad974ff9b02535:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:16:23,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:23,870 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36102 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:23,870 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:23,870 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 77e3e5908c8d586f7cad974ff9b02535/A is initiating minor compaction (all files) 2024-11-16T10:16:23,870 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 77e3e5908c8d586f7cad974ff9b02535/B is initiating minor compaction (all files) 2024-11-16T10:16:23,870 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77e3e5908c8d586f7cad974ff9b02535/B in TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:23,870 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77e3e5908c8d586f7cad974ff9b02535/A in TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:23,871 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/f54f9d722a98451a96f5a9448d74b2ec, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/2c831c8721424193b9098c21e1702d26, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/54435b39ec79438aad50d72494957363] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp, totalSize=35.3 K 2024-11-16T10:16:23,871 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/c32445e4762a4bb5a6a3fc991ac567ed, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/e5c1918b1e66402181b601e1158bbe88, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/28348b7200714fad87cad418c32ab69b] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp, totalSize=33.0 K 2024-11-16T10:16:23,871 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting c32445e4762a4bb5a6a3fc991ac567ed, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731752179719 2024-11-16T10:16:23,871 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting f54f9d722a98451a96f5a9448d74b2ec, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731752179719 2024-11-16T10:16:23,871 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting e5c1918b1e66402181b601e1158bbe88, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731752180815 2024-11-16T10:16:23,871 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c831c8721424193b9098c21e1702d26, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731752180815 2024-11-16T10:16:23,871 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 28348b7200714fad87cad418c32ab69b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731752181937 2024-11-16T10:16:23,872 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54435b39ec79438aad50d72494957363, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731752181937 2024-11-16T10:16:23,878 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77e3e5908c8d586f7cad974ff9b02535#B#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:23,878 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/9da18e2f7d0f45828df1e6d5bbf8f4a0 is 50, key is test_row_0/B:col10/1731752183016/Put/seqid=0 2024-11-16T10:16:23,881 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77e3e5908c8d586f7cad974ff9b02535#A#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:23,881 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/1b68a554aa8e44239ad59e93e4ca2f6c is 50, key is test_row_0/A:col10/1731752183016/Put/seqid=0 2024-11-16T10:16:23,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742295_1471 (size=12207) 2024-11-16T10:16:23,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742296_1472 (size=12207) 2024-11-16T10:16:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:24,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77e3e5908c8d586f7cad974ff9b02535 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T10:16:24,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=A 2024-11-16T10:16:24,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:24,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=B 2024-11-16T10:16:24,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:24,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=C 2024-11-16T10:16:24,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:24,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/4e59aaa3369048d194dfdf1337ef2a08 is 50, key is test_row_0/A:col10/1731752183070/Put/seqid=0 2024-11-16T10:16:24,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742297_1473 (size=14341) 2024-11-16T10:16:24,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752244138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:24,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:24,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752244139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:24,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:24,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752244140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:24,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:24,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:24,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51766 deadline: 1731752244244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:24,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:24,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1731752244254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752244254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752244254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:24,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:24,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752244264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:24,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:24,287 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/9da18e2f7d0f45828df1e6d5bbf8f4a0 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/9da18e2f7d0f45828df1e6d5bbf8f4a0 2024-11-16T10:16:24,288 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/1b68a554aa8e44239ad59e93e4ca2f6c as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/1b68a554aa8e44239ad59e93e4ca2f6c 2024-11-16T10:16:24,291 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77e3e5908c8d586f7cad974ff9b02535/B of 77e3e5908c8d586f7cad974ff9b02535 into 9da18e2f7d0f45828df1e6d5bbf8f4a0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:24,291 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:24,291 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., storeName=77e3e5908c8d586f7cad974ff9b02535/B, priority=13, startTime=1731752183870; duration=0sec 2024-11-16T10:16:24,291 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:24,291 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77e3e5908c8d586f7cad974ff9b02535:B 2024-11-16T10:16:24,291 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:24,292 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77e3e5908c8d586f7cad974ff9b02535/A of 77e3e5908c8d586f7cad974ff9b02535 into 1b68a554aa8e44239ad59e93e4ca2f6c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:24,292 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:24,292 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., storeName=77e3e5908c8d586f7cad974ff9b02535/A, priority=13, startTime=1731752183870; duration=0sec 2024-11-16T10:16:24,292 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:24,292 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77e3e5908c8d586f7cad974ff9b02535:A 2024-11-16T10:16:24,292 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:24,292 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 77e3e5908c8d586f7cad974ff9b02535/C is initiating minor compaction (all files) 2024-11-16T10:16:24,292 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77e3e5908c8d586f7cad974ff9b02535/C in TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:24,292 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/93ffddd471984f0cad08134b05f37500, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/f7e294aa632a4d8b8cf313fe977d1112, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/5e04f8849bb24646a3cbb4080792e154] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp, totalSize=33.0 K 2024-11-16T10:16:24,292 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 93ffddd471984f0cad08134b05f37500, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731752179719 2024-11-16T10:16:24,293 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting f7e294aa632a4d8b8cf313fe977d1112, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731752180815 2024-11-16T10:16:24,293 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e04f8849bb24646a3cbb4080792e154, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731752181937 2024-11-16T10:16:24,298 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77e3e5908c8d586f7cad974ff9b02535#C#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:24,299 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/489f8f6ad0e14b219bf30a6b931585a6 is 50, key is test_row_0/C:col10/1731752183016/Put/seqid=0 2024-11-16T10:16:24,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742298_1474 (size=12207) 2024-11-16T10:16:24,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752244464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:24,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752244464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:24,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:24,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:24,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752244484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:24,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/4e59aaa3369048d194dfdf1337ef2a08 2024-11-16T10:16:24,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/34683eb08b844576b36e8e7c6a9cca51 is 50, key is test_row_0/B:col10/1731752183070/Put/seqid=0 2024-11-16T10:16:24,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742299_1475 (size=12001) 2024-11-16T10:16:24,706 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/489f8f6ad0e14b219bf30a6b931585a6 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/489f8f6ad0e14b219bf30a6b931585a6 2024-11-16T10:16:24,710 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77e3e5908c8d586f7cad974ff9b02535/C of 77e3e5908c8d586f7cad974ff9b02535 into 489f8f6ad0e14b219bf30a6b931585a6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:24,710 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:24,710 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., storeName=77e3e5908c8d586f7cad974ff9b02535/C, priority=13, startTime=1731752183870; duration=0sec 2024-11-16T10:16:24,710 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:24,710 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77e3e5908c8d586f7cad974ff9b02535:C 2024-11-16T10:16:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T10:16:24,774 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:24,774 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:16:24,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-16T10:16:24,776 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:16:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T10:16:24,776 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:16:24,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:16:24,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752244774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:24,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:24,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752244774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:24,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:24,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752244794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:24,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:24,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T10:16:24,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/34683eb08b844576b36e8e7c6a9cca51 2024-11-16T10:16:24,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T10:16:24,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:24,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:24,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:24,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:24,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:24,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:24,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/aa9d4691e8ab44afbdc2b957fb928ec1 is 50, key is test_row_0/C:col10/1731752183070/Put/seqid=0 2024-11-16T10:16:24,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742300_1476 (size=12001) 2024-11-16T10:16:25,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T10:16:25,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:25,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:25,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:25,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:25,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:25,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T10:16:25,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T10:16:25,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:25,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:25,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:25,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:25,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:25,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:25,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:25,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51766 deadline: 1731752245264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:25,268 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_0' in 77e3e5908c8d586f7cad974ff9b02535 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:16:25,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:25,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:25,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:25,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:25,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1731752245274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:25,277 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in 77e3e5908c8d586f7cad974ff9b02535 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:16:25,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:25,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:25,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:25,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:25,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752245283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:25,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:25,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:25,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:25,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:25,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752245294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:25,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:25,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:25,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:25,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:25,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752245304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:25,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:25,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:25,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:25,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/aa9d4691e8ab44afbdc2b957fb928ec1 2024-11-16T10:16:25,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/4e59aaa3369048d194dfdf1337ef2a08 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/4e59aaa3369048d194dfdf1337ef2a08 2024-11-16T10:16:25,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/4e59aaa3369048d194dfdf1337ef2a08, entries=200, sequenceid=114, filesize=14.0 K 2024-11-16T10:16:25,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/34683eb08b844576b36e8e7c6a9cca51 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/34683eb08b844576b36e8e7c6a9cca51 2024-11-16T10:16:25,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/34683eb08b844576b36e8e7c6a9cca51, entries=150, sequenceid=114, filesize=11.7 K 2024-11-16T10:16:25,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/aa9d4691e8ab44afbdc2b957fb928ec1 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/aa9d4691e8ab44afbdc2b957fb928ec1 2024-11-16T10:16:25,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/aa9d4691e8ab44afbdc2b957fb928ec1, entries=150, sequenceid=114, filesize=11.7 K 2024-11-16T10:16:25,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 77e3e5908c8d586f7cad974ff9b02535 in 1256ms, sequenceid=114, compaction requested=false 2024-11-16T10:16:25,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:25,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T10:16:25,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:25,386 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 77e3e5908c8d586f7cad974ff9b02535 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T10:16:25,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=A 2024-11-16T10:16:25,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:25,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=B 2024-11-16T10:16:25,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:25,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=C 2024-11-16T10:16:25,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:25,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/887a2fd19e4c4773ac5b2f4ce80774af is 50, key is test_row_0/A:col10/1731752184135/Put/seqid=0 2024-11-16T10:16:25,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742301_1477 (size=12001) 2024-11-16T10:16:25,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T10:16:25,800 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/887a2fd19e4c4773ac5b2f4ce80774af 2024-11-16T10:16:25,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/e77c51f34c504010b93189faa04539ed is 50, key is test_row_0/B:col10/1731752184135/Put/seqid=0 2024-11-16T10:16:25,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742302_1478 (size=12001) 2024-11-16T10:16:25,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T10:16:26,220 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/e77c51f34c504010b93189faa04539ed 2024-11-16T10:16:26,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/ae32e20122bc44dd8793b65d5ed4c8b4 is 50, key is test_row_0/C:col10/1731752184135/Put/seqid=0 2024-11-16T10:16:26,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742303_1479 (size=12001) 2024-11-16T10:16:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:26,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:26,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752246359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:26,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:26,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752246361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:26,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:26,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752246362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:26,371 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,371 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,371 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:26,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752246464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:26,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:26,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752246474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:26,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:26,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752246474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:26,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:26,645 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/ae32e20122bc44dd8793b65d5ed4c8b4 2024-11-16T10:16:26,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/887a2fd19e4c4773ac5b2f4ce80774af as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/887a2fd19e4c4773ac5b2f4ce80774af 2024-11-16T10:16:26,650 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/887a2fd19e4c4773ac5b2f4ce80774af, entries=150, sequenceid=129, filesize=11.7 K 2024-11-16T10:16:26,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/e77c51f34c504010b93189faa04539ed as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/e77c51f34c504010b93189faa04539ed 2024-11-16T10:16:26,653 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/e77c51f34c504010b93189faa04539ed, entries=150, sequenceid=129, filesize=11.7 K 2024-11-16T10:16:26,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/ae32e20122bc44dd8793b65d5ed4c8b4 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/ae32e20122bc44dd8793b65d5ed4c8b4 2024-11-16T10:16:26,656 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/ae32e20122bc44dd8793b65d5ed4c8b4, entries=150, sequenceid=129, filesize=11.7 K 2024-11-16T10:16:26,657 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 77e3e5908c8d586f7cad974ff9b02535 in 1271ms, sequenceid=129, compaction requested=true 2024-11-16T10:16:26,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:26,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:26,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-16T10:16:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-16T10:16:26,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-16T10:16:26,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8820 sec 2024-11-16T10:16:26,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.8860 sec 2024-11-16T10:16:26,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77e3e5908c8d586f7cad974ff9b02535 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-16T10:16:26,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=A 2024-11-16T10:16:26,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:26,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=B 2024-11-16T10:16:26,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:26,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=C 2024-11-16T10:16:26,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:26,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/1f9175b7ae794de0aa53a4453956e7dc is 50, key is test_row_0/A:col10/1731752186687/Put/seqid=0 2024-11-16T10:16:26,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742304_1480 (size=14541) 2024-11-16T10:16:26,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/1f9175b7ae794de0aa53a4453956e7dc 2024-11-16T10:16:26,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/982cfa83804044f28ba1a64b1765a897 is 50, key is test_row_0/B:col10/1731752186687/Put/seqid=0 2024-11-16T10:16:26,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742305_1481 (size=12151) 2024-11-16T10:16:26,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/982cfa83804044f28ba1a64b1765a897 2024-11-16T10:16:26,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/e24999964eb34cb99ea91e9d978a8f13 is 50, key is test_row_0/C:col10/1731752186687/Put/seqid=0 2024-11-16T10:16:26,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742306_1482 (size=12151) 2024-11-16T10:16:26,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752246709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:26,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:26,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/e24999964eb34cb99ea91e9d978a8f13 2024-11-16T10:16:26,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752246710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:26,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:26,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752246720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:26,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/1f9175b7ae794de0aa53a4453956e7dc as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/1f9175b7ae794de0aa53a4453956e7dc 2024-11-16T10:16:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:26,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/1f9175b7ae794de0aa53a4453956e7dc, entries=200, sequenceid=150, filesize=14.2 K 2024-11-16T10:16:26,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/982cfa83804044f28ba1a64b1765a897 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/982cfa83804044f28ba1a64b1765a897 2024-11-16T10:16:26,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/982cfa83804044f28ba1a64b1765a897, entries=150, sequenceid=150, filesize=11.9 K 2024-11-16T10:16:26,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/e24999964eb34cb99ea91e9d978a8f13 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/e24999964eb34cb99ea91e9d978a8f13 2024-11-16T10:16:26,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/e24999964eb34cb99ea91e9d978a8f13, entries=150, sequenceid=150, filesize=11.9 K 2024-11-16T10:16:26,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 77e3e5908c8d586f7cad974ff9b02535 in 46ms, sequenceid=150, compaction requested=true 2024-11-16T10:16:26,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:26,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77e3e5908c8d586f7cad974ff9b02535:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:16:26,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:26,734 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:16:26,734 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:16:26,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77e3e5908c8d586f7cad974ff9b02535:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:16:26,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:26,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77e3e5908c8d586f7cad974ff9b02535:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:16:26,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:26,744 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:16:26,744 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:16:26,744 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 77e3e5908c8d586f7cad974ff9b02535/A is initiating minor compaction (all files) 2024-11-16T10:16:26,744 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 77e3e5908c8d586f7cad974ff9b02535/B is initiating minor compaction (all files) 2024-11-16T10:16:26,744 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77e3e5908c8d586f7cad974ff9b02535/A in TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:26,744 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77e3e5908c8d586f7cad974ff9b02535/B in TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:26,744 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/1b68a554aa8e44239ad59e93e4ca2f6c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/4e59aaa3369048d194dfdf1337ef2a08, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/887a2fd19e4c4773ac5b2f4ce80774af, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/1f9175b7ae794de0aa53a4453956e7dc] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp, totalSize=51.8 K 2024-11-16T10:16:26,744 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/9da18e2f7d0f45828df1e6d5bbf8f4a0, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/34683eb08b844576b36e8e7c6a9cca51, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/e77c51f34c504010b93189faa04539ed, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/982cfa83804044f28ba1a64b1765a897] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp, totalSize=47.2 K 2024-11-16T10:16:26,745 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 9da18e2f7d0f45828df1e6d5bbf8f4a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731752180821 2024-11-16T10:16:26,745 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 34683eb08b844576b36e8e7c6a9cca51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731752183070 2024-11-16T10:16:26,745 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b68a554aa8e44239ad59e93e4ca2f6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731752180821 2024-11-16T10:16:26,745 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting e77c51f34c504010b93189faa04539ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731752184135 2024-11-16T10:16:26,745 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e59aaa3369048d194dfdf1337ef2a08, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731752183070 2024-11-16T10:16:26,745 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 982cfa83804044f28ba1a64b1765a897, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731752186358 2024-11-16T10:16:26,745 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 887a2fd19e4c4773ac5b2f4ce80774af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731752184135 2024-11-16T10:16:26,746 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f9175b7ae794de0aa53a4453956e7dc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731752186358 2024-11-16T10:16:26,752 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77e3e5908c8d586f7cad974ff9b02535#B#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:26,752 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/98956f8d67f9475f9fb3572b3e1b41e1 is 50, key is test_row_0/B:col10/1731752186687/Put/seqid=0 2024-11-16T10:16:26,753 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77e3e5908c8d586f7cad974ff9b02535#A#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:26,753 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/4fbd43942c4f4072a97d07f34af5df2f is 50, key is test_row_0/A:col10/1731752186687/Put/seqid=0 2024-11-16T10:16:26,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742308_1484 (size=12493) 2024-11-16T10:16:26,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742307_1483 (size=12493) 2024-11-16T10:16:26,767 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/98956f8d67f9475f9fb3572b3e1b41e1 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/98956f8d67f9475f9fb3572b3e1b41e1 2024-11-16T10:16:26,769 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/4fbd43942c4f4072a97d07f34af5df2f as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/4fbd43942c4f4072a97d07f34af5df2f 2024-11-16T10:16:26,771 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77e3e5908c8d586f7cad974ff9b02535/B of 77e3e5908c8d586f7cad974ff9b02535 into 98956f8d67f9475f9fb3572b3e1b41e1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:26,771 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:26,771 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., storeName=77e3e5908c8d586f7cad974ff9b02535/B, priority=12, startTime=1731752186734; duration=0sec 2024-11-16T10:16:26,771 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:26,771 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77e3e5908c8d586f7cad974ff9b02535:B 2024-11-16T10:16:26,771 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:16:26,773 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:16:26,773 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 77e3e5908c8d586f7cad974ff9b02535/C is initiating minor compaction (all files) 2024-11-16T10:16:26,773 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77e3e5908c8d586f7cad974ff9b02535/C in TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:26,774 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/489f8f6ad0e14b219bf30a6b931585a6, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/aa9d4691e8ab44afbdc2b957fb928ec1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/ae32e20122bc44dd8793b65d5ed4c8b4, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/e24999964eb34cb99ea91e9d978a8f13] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp, totalSize=47.2 K 2024-11-16T10:16:26,774 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 489f8f6ad0e14b219bf30a6b931585a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731752180821 2024-11-16T10:16:26,774 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting aa9d4691e8ab44afbdc2b957fb928ec1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731752183070 2024-11-16T10:16:26,775 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting ae32e20122bc44dd8793b65d5ed4c8b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731752184135 2024-11-16T10:16:26,775 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting e24999964eb34cb99ea91e9d978a8f13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731752186358 2024-11-16T10:16:26,775 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77e3e5908c8d586f7cad974ff9b02535/A of 77e3e5908c8d586f7cad974ff9b02535 into 4fbd43942c4f4072a97d07f34af5df2f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:26,775 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:26,775 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., storeName=77e3e5908c8d586f7cad974ff9b02535/A, priority=12, startTime=1731752186734; duration=0sec 2024-11-16T10:16:26,775 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:26,775 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77e3e5908c8d586f7cad974ff9b02535:A 2024-11-16T10:16:26,787 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77e3e5908c8d586f7cad974ff9b02535#C#compaction#410 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:26,787 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/344516967b9748e6a991cc56480e76ab is 50, key is test_row_0/C:col10/1731752186687/Put/seqid=0 2024-11-16T10:16:26,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742309_1485 (size=12493) 2024-11-16T10:16:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:26,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77e3e5908c8d586f7cad974ff9b02535 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T10:16:26,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=A 2024-11-16T10:16:26,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:26,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=B 2024-11-16T10:16:26,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:26,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=C 2024-11-16T10:16:26,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:26,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/6f54ad978bcc475b8416e4ffd86b997c is 50, key is test_row_0/A:col10/1731752186707/Put/seqid=0 2024-11-16T10:16:26,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742310_1486 (size=14541) 2024-11-16T10:16:26,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752246878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:26,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:26,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752246882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:26,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:26,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:26,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752246886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:26,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:26,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T10:16:26,924 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:26,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:16:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-16T10:16:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T10:16:26,926 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:16:26,927 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:16:26,927 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:16:27,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752246994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:27,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752246994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:27,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:27,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:27,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:27,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:27,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:27,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:27,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752247004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:27,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:27,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:27,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T10:16:27,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T10:16:27,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:27,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:27,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:27,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:27,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:27,204 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/344516967b9748e6a991cc56480e76ab as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/344516967b9748e6a991cc56480e76ab 2024-11-16T10:16:27,208 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77e3e5908c8d586f7cad974ff9b02535/C of 77e3e5908c8d586f7cad974ff9b02535 into 344516967b9748e6a991cc56480e76ab(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:27,208 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:27,208 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., storeName=77e3e5908c8d586f7cad974ff9b02535/C, priority=12, startTime=1731752186735; duration=0sec 2024-11-16T10:16:27,208 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:27,208 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77e3e5908c8d586f7cad974ff9b02535:C 2024-11-16T10:16:27,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752247204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:27,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:27,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:27,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752247204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:27,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:27,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:27,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:27,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:27,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T10:16:27,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:27,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:27,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:27,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:27,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:27,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752247224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:27,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:27,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:27,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:27,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/6f54ad978bcc475b8416e4ffd86b997c 2024-11-16T10:16:27,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/5466b95fa57a4cb68a72c9c5005ac9e5 is 50, key is test_row_0/B:col10/1731752186707/Put/seqid=0 2024-11-16T10:16:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T10:16:27,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742311_1487 (size=12151) 2024-11-16T10:16:27,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51766 deadline: 1731752247274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:27,278 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_0' in 77e3e5908c8d586f7cad974ff9b02535 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:16:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:27,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1731752247284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:27,289 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in 77e3e5908c8d586f7cad974ff9b02535 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:16:27,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:27,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:27,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:27,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T10:16:27,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:27,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:27,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:27,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:27,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:27,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:27,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752247524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:27,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752247524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:27,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:27,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:27,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:27,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:27,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:27,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:27,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T10:16:27,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:27,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:27,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:27,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:27,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:27,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:27,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752247544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T10:16:27,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/5466b95fa57a4cb68a72c9c5005ac9e5 2024-11-16T10:16:27,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/90bad977472b49ba9d1a6932a4e302c9 is 50, key is test_row_0/C:col10/1731752186707/Put/seqid=0 2024-11-16T10:16:27,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742312_1488 (size=12151) 2024-11-16T10:16:27,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T10:16:27,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:27,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:27,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:27,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:27,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:27,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T10:16:27,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:27,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:27,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:27,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:27,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:27,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T10:16:27,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:27,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:27,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:27,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:27,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:28,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752248034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:28,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:28,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:28,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:28,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752248044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:28,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:28,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:28,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:28,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/90bad977472b49ba9d1a6932a4e302c9 2024-11-16T10:16:28,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/6f54ad978bcc475b8416e4ffd86b997c as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/6f54ad978bcc475b8416e4ffd86b997c 2024-11-16T10:16:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T10:16:28,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/6f54ad978bcc475b8416e4ffd86b997c, entries=200, sequenceid=164, filesize=14.2 K 2024-11-16T10:16:28,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/5466b95fa57a4cb68a72c9c5005ac9e5 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/5466b95fa57a4cb68a72c9c5005ac9e5 2024-11-16T10:16:28,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/5466b95fa57a4cb68a72c9c5005ac9e5, entries=150, sequenceid=164, filesize=11.9 K 2024-11-16T10:16:28,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/90bad977472b49ba9d1a6932a4e302c9 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/90bad977472b49ba9d1a6932a4e302c9 2024-11-16T10:16:28,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752248064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:28,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:28,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:28,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:28,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/90bad977472b49ba9d1a6932a4e302c9, entries=150, sequenceid=164, filesize=11.9 K 2024-11-16T10:16:28,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 77e3e5908c8d586f7cad974ff9b02535 in 1246ms, sequenceid=164, compaction requested=false 2024-11-16T10:16:28,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:28,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T10:16:28,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:28,148 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 77e3e5908c8d586f7cad974ff9b02535 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T10:16:28,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=A 2024-11-16T10:16:28,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:28,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=B 2024-11-16T10:16:28,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:28,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=C 2024-11-16T10:16:28,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:28,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/24df0799caae43138d9abe31ea1ec61f is 50, key is test_row_0/A:col10/1731752186885/Put/seqid=0 2024-11-16T10:16:28,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742313_1489 (size=12151) 2024-11-16T10:16:28,555 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/24df0799caae43138d9abe31ea1ec61f 2024-11-16T10:16:28,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/c2677db0d82c43218f6a8934e7d37049 is 50, key is test_row_0/B:col10/1731752186885/Put/seqid=0 2024-11-16T10:16:28,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742314_1490 (size=12151) 2024-11-16T10:16:28,966 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/c2677db0d82c43218f6a8934e7d37049 2024-11-16T10:16:28,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/29881b5c0dba40f59b88467cb9fc7b2d is 50, key is test_row_0/C:col10/1731752186885/Put/seqid=0 2024-11-16T10:16:28,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742315_1491 (size=12151) 2024-11-16T10:16:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:29,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T10:16:29,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752249080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:29,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:29,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752249082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:29,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:29,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:29,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752249082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:29,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752249194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752249194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:29,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752249204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:29,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:29,383 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/29881b5c0dba40f59b88467cb9fc7b2d 2024-11-16T10:16:29,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/24df0799caae43138d9abe31ea1ec61f as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/24df0799caae43138d9abe31ea1ec61f 2024-11-16T10:16:29,389 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/24df0799caae43138d9abe31ea1ec61f, entries=150, sequenceid=185, filesize=11.9 K 2024-11-16T10:16:29,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/c2677db0d82c43218f6a8934e7d37049 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/c2677db0d82c43218f6a8934e7d37049 2024-11-16T10:16:29,392 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/c2677db0d82c43218f6a8934e7d37049, entries=150, sequenceid=185, filesize=11.9 K 2024-11-16T10:16:29,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/29881b5c0dba40f59b88467cb9fc7b2d as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/29881b5c0dba40f59b88467cb9fc7b2d 2024-11-16T10:16:29,395 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/29881b5c0dba40f59b88467cb9fc7b2d, entries=150, sequenceid=185, filesize=11.9 K 2024-11-16T10:16:29,396 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 77e3e5908c8d586f7cad974ff9b02535 in 1248ms, sequenceid=185, compaction requested=true 2024-11-16T10:16:29,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:29,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:29,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-16T10:16:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-16T10:16:29,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-16T10:16:29,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4700 sec 2024-11-16T10:16:29,401 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.4750 sec 2024-11-16T10:16:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:29,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77e3e5908c8d586f7cad974ff9b02535 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T10:16:29,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=A 2024-11-16T10:16:29,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:29,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=B 2024-11-16T10:16:29,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:29,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=C 2024-11-16T10:16:29,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:29,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/03da6bdd0c7a49799da88f2a48916d56 is 50, key is test_row_0/A:col10/1731752189408/Put/seqid=0 2024-11-16T10:16:29,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742316_1492 (size=14541) 2024-11-16T10:16:29,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/03da6bdd0c7a49799da88f2a48916d56 2024-11-16T10:16:29,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/9a22d625e27546b0993c89e00011e82b is 50, key is test_row_0/B:col10/1731752189408/Put/seqid=0 2024-11-16T10:16:29,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742317_1493 (size=12151) 2024-11-16T10:16:29,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752249474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:29,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:29,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752249476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:29,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752249477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:29,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:29,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752249584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:29,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:29,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752249594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:29,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:29,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752249604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:29,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:29,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752249804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:29,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:29,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:29,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752249804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:29,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:29,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752249824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:29,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:29,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:29,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/9a22d625e27546b0993c89e00011e82b 2024-11-16T10:16:29,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/d479a914703741a3ad874288d127ff71 is 50, key is test_row_0/C:col10/1731752189408/Put/seqid=0 2024-11-16T10:16:29,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742318_1494 (size=12151) 2024-11-16T10:16:30,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752250114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:30,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:30,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:30,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:30,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752250123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:30,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:30,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752250144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:30,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:30,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:30,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:30,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/d479a914703741a3ad874288d127ff71 2024-11-16T10:16:30,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/03da6bdd0c7a49799da88f2a48916d56 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/03da6bdd0c7a49799da88f2a48916d56 2024-11-16T10:16:30,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/03da6bdd0c7a49799da88f2a48916d56, entries=200, sequenceid=198, filesize=14.2 K 2024-11-16T10:16:30,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/9a22d625e27546b0993c89e00011e82b as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/9a22d625e27546b0993c89e00011e82b 2024-11-16T10:16:30,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/9a22d625e27546b0993c89e00011e82b, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T10:16:30,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/d479a914703741a3ad874288d127ff71 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/d479a914703741a3ad874288d127ff71 2024-11-16T10:16:30,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/d479a914703741a3ad874288d127ff71, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T10:16:30,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 77e3e5908c8d586f7cad974ff9b02535 in 858ms, sequenceid=198, compaction requested=true 2024-11-16T10:16:30,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:30,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77e3e5908c8d586f7cad974ff9b02535:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:16:30,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:30,267 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:16:30,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77e3e5908c8d586f7cad974ff9b02535:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:16:30,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:30,267 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:16:30,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77e3e5908c8d586f7cad974ff9b02535:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:16:30,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:30,268 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:16:30,268 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:16:30,268 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 77e3e5908c8d586f7cad974ff9b02535/A is initiating minor compaction (all files) 2024-11-16T10:16:30,268 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 77e3e5908c8d586f7cad974ff9b02535/B is initiating minor compaction (all files) 2024-11-16T10:16:30,268 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77e3e5908c8d586f7cad974ff9b02535/A in TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:30,268 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77e3e5908c8d586f7cad974ff9b02535/B in TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:30,268 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/4fbd43942c4f4072a97d07f34af5df2f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/6f54ad978bcc475b8416e4ffd86b997c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/24df0799caae43138d9abe31ea1ec61f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/03da6bdd0c7a49799da88f2a48916d56] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp, totalSize=52.5 K 2024-11-16T10:16:30,268 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/98956f8d67f9475f9fb3572b3e1b41e1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/5466b95fa57a4cb68a72c9c5005ac9e5, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/c2677db0d82c43218f6a8934e7d37049, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/9a22d625e27546b0993c89e00011e82b] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp, totalSize=47.8 K 2024-11-16T10:16:30,269 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fbd43942c4f4072a97d07f34af5df2f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731752186358 2024-11-16T10:16:30,269 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 98956f8d67f9475f9fb3572b3e1b41e1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731752186358 2024-11-16T10:16:30,269 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f54ad978bcc475b8416e4ffd86b997c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731752186691 2024-11-16T10:16:30,269 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 5466b95fa57a4cb68a72c9c5005ac9e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731752186707 2024-11-16T10:16:30,269 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24df0799caae43138d9abe31ea1ec61f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731752186854 2024-11-16T10:16:30,269 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting c2677db0d82c43218f6a8934e7d37049, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731752186854 2024-11-16T10:16:30,269 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a22d625e27546b0993c89e00011e82b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731752189069 2024-11-16T10:16:30,269 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03da6bdd0c7a49799da88f2a48916d56, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731752189069 2024-11-16T10:16:30,276 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77e3e5908c8d586f7cad974ff9b02535#B#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:30,276 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77e3e5908c8d586f7cad974ff9b02535#A#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:30,276 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/19eeb9a080044da8a53fd60b1e6907b1 is 50, key is test_row_0/B:col10/1731752189408/Put/seqid=0 2024-11-16T10:16:30,277 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/c3dfc6f895bc46129dadfac15d05c062 is 50, key is test_row_0/A:col10/1731752189408/Put/seqid=0 2024-11-16T10:16:30,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742319_1495 (size=12629) 2024-11-16T10:16:30,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742320_1496 (size=12629) 2024-11-16T10:16:30,288 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/19eeb9a080044da8a53fd60b1e6907b1 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/19eeb9a080044da8a53fd60b1e6907b1 2024-11-16T10:16:30,292 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77e3e5908c8d586f7cad974ff9b02535/B of 77e3e5908c8d586f7cad974ff9b02535 into 19eeb9a080044da8a53fd60b1e6907b1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:30,293 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:30,293 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., storeName=77e3e5908c8d586f7cad974ff9b02535/B, priority=12, startTime=1731752190267; duration=0sec 2024-11-16T10:16:30,293 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:30,293 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77e3e5908c8d586f7cad974ff9b02535:B 2024-11-16T10:16:30,293 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:16:30,293 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:16:30,294 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 77e3e5908c8d586f7cad974ff9b02535/C is initiating minor compaction (all files) 2024-11-16T10:16:30,294 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77e3e5908c8d586f7cad974ff9b02535/C in TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:30,294 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/344516967b9748e6a991cc56480e76ab, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/90bad977472b49ba9d1a6932a4e302c9, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/29881b5c0dba40f59b88467cb9fc7b2d, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/d479a914703741a3ad874288d127ff71] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp, totalSize=47.8 K 2024-11-16T10:16:30,294 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 344516967b9748e6a991cc56480e76ab, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731752186358 2024-11-16T10:16:30,294 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 90bad977472b49ba9d1a6932a4e302c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731752186707 2024-11-16T10:16:30,294 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 29881b5c0dba40f59b88467cb9fc7b2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731752186854 2024-11-16T10:16:30,294 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting d479a914703741a3ad874288d127ff71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731752189069 2024-11-16T10:16:30,299 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77e3e5908c8d586f7cad974ff9b02535#C#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:30,300 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/09e7c93ff5a046bbaa900b59fa200c56 is 50, key is test_row_0/C:col10/1731752189408/Put/seqid=0 2024-11-16T10:16:30,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742321_1497 (size=12629) 2024-11-16T10:16:30,311 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/09e7c93ff5a046bbaa900b59fa200c56 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/09e7c93ff5a046bbaa900b59fa200c56 2024-11-16T10:16:30,314 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77e3e5908c8d586f7cad974ff9b02535/C of 77e3e5908c8d586f7cad974ff9b02535 into 09e7c93ff5a046bbaa900b59fa200c56(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:30,314 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:30,314 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., storeName=77e3e5908c8d586f7cad974ff9b02535/C, priority=12, startTime=1731752190267; duration=0sec 2024-11-16T10:16:30,315 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:30,315 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77e3e5908c8d586f7cad974ff9b02535:C 2024-11-16T10:16:30,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:30,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77e3e5908c8d586f7cad974ff9b02535 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T10:16:30,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=A 2024-11-16T10:16:30,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:30,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=B 2024-11-16T10:16:30,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:30,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=C 2024-11-16T10:16:30,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:30,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/a65d9a96502a427e958172f71298f934 is 50, key is test_row_0/A:col10/1731752189457/Put/seqid=0 2024-11-16T10:16:30,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742322_1498 (size=14541) 2024-11-16T10:16:30,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:30,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752250668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:30,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:30,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:30,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:30,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752250669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:30,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752250670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:30,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:30,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:30,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:30,683 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/c3dfc6f895bc46129dadfac15d05c062 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/c3dfc6f895bc46129dadfac15d05c062 2024-11-16T10:16:30,687 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77e3e5908c8d586f7cad974ff9b02535/A of 77e3e5908c8d586f7cad974ff9b02535 into c3dfc6f895bc46129dadfac15d05c062(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:30,687 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:30,687 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., storeName=77e3e5908c8d586f7cad974ff9b02535/A, priority=12, startTime=1731752190267; duration=0sec 2024-11-16T10:16:30,687 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:30,687 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77e3e5908c8d586f7cad974ff9b02535:A 2024-11-16T10:16:30,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752250784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:30,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:30,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752250784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:30,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:30,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:30,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:30,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:30,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:30,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:30,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752250784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:30,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:30,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:30,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:31,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752250993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:31,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:31,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752251004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:31,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:31,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:31,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:31,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752251004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:31,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:31,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:31,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:31,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/a65d9a96502a427e958172f71298f934 2024-11-16T10:16:31,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/8f38e11689f54e1a8860bf4d3da7de31 is 50, key is test_row_0/B:col10/1731752189457/Put/seqid=0 2024-11-16T10:16:31,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742323_1499 (size=12151) 2024-11-16T10:16:31,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/8f38e11689f54e1a8860bf4d3da7de31 2024-11-16T10:16:31,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/8bed6301eac548ee85c8e4f4ac0437bc is 50, key is test_row_0/C:col10/1731752189457/Put/seqid=0 2024-11-16T10:16:31,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742324_1500 (size=12151) 2024-11-16T10:16:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T10:16:31,094 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:31,094 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:16:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-16T10:16:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T10:16:31,096 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:16:31,096 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:16:31,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:16:31,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T10:16:31,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T10:16:31,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:31,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:31,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:31,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:31,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:31,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:31,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1731752251304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:31,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:31,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752251304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:31,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:31,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:31,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:31,309 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in 77e3e5908c8d586f7cad974ff9b02535 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:16:31,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:31,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:31,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:31,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51766 deadline: 1731752251313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:31,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752251314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:31,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:31,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:31,319 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_0' in 77e3e5908c8d586f7cad974ff9b02535 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:16:31,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:31,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:31,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:31,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:31,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752251324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:31,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:31,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:31,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:31,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T10:16:31,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:31,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:31,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:31,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:31,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T10:16:31,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/8bed6301eac548ee85c8e4f4ac0437bc 2024-11-16T10:16:31,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/a65d9a96502a427e958172f71298f934 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/a65d9a96502a427e958172f71298f934 2024-11-16T10:16:31,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/a65d9a96502a427e958172f71298f934, entries=200, sequenceid=220, filesize=14.2 K 2024-11-16T10:16:31,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/8f38e11689f54e1a8860bf4d3da7de31 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/8f38e11689f54e1a8860bf4d3da7de31 2024-11-16T10:16:31,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/8f38e11689f54e1a8860bf4d3da7de31, entries=150, sequenceid=220, filesize=11.9 K 2024-11-16T10:16:31,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/8bed6301eac548ee85c8e4f4ac0437bc as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/8bed6301eac548ee85c8e4f4ac0437bc 2024-11-16T10:16:31,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/8bed6301eac548ee85c8e4f4ac0437bc, entries=150, sequenceid=220, filesize=11.9 K 2024-11-16T10:16:31,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 77e3e5908c8d586f7cad974ff9b02535 in 843ms, sequenceid=220, compaction requested=false 2024-11-16T10:16:31,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:31,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T10:16:31,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:31,553 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 77e3e5908c8d586f7cad974ff9b02535 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T10:16:31,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=A 2024-11-16T10:16:31,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:31,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=B 2024-11-16T10:16:31,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:31,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=C 2024-11-16T10:16:31,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:31,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/90489a00731c496ca38cb0249cbb521b is 50, key is test_row_0/A:col10/1731752190666/Put/seqid=0 2024-11-16T10:16:31,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742325_1501 (size=12151) 2024-11-16T10:16:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T10:16:31,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:31,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:31,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:31,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752251878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:31,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:31,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752251886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:31,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:31,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752251887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:31,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:31,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:31,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:31,963 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/90489a00731c496ca38cb0249cbb521b 2024-11-16T10:16:31,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/9bb16961a2cd40a583536ec47eadf41c is 50, key is test_row_0/B:col10/1731752190666/Put/seqid=0 2024-11-16T10:16:31,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742326_1502 (size=12151) 2024-11-16T10:16:32,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:32,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752252004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:32,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:32,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:32,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752252004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:32,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:32,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:32,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:32,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:32,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:32,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752252005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:32,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:32,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:32,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:32,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:32,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:32,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752252224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:32,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:32,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752252224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:32,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:32,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752252224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:32,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T10:16:32,373 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/9bb16961a2cd40a583536ec47eadf41c 2024-11-16T10:16:32,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/890fd37f70dc415292da867ee3388014 is 50, key is test_row_0/C:col10/1731752190666/Put/seqid=0 2024-11-16T10:16:32,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742327_1503 (size=12151) 2024-11-16T10:16:32,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:32,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752252534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:32,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:32,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:32,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752252534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:32,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:32,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:32,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:32,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:32,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:32,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:32,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752252534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:32,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:32,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:32,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:32,782 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/890fd37f70dc415292da867ee3388014 2024-11-16T10:16:32,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/90489a00731c496ca38cb0249cbb521b as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/90489a00731c496ca38cb0249cbb521b 2024-11-16T10:16:32,788 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/90489a00731c496ca38cb0249cbb521b, entries=150, sequenceid=233, filesize=11.9 K 2024-11-16T10:16:32,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/9bb16961a2cd40a583536ec47eadf41c as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/9bb16961a2cd40a583536ec47eadf41c 2024-11-16T10:16:32,792 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/9bb16961a2cd40a583536ec47eadf41c, entries=150, sequenceid=233, filesize=11.9 K 2024-11-16T10:16:32,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/890fd37f70dc415292da867ee3388014 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/890fd37f70dc415292da867ee3388014 2024-11-16T10:16:32,796 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/890fd37f70dc415292da867ee3388014, entries=150, sequenceid=233, filesize=11.9 K 2024-11-16T10:16:32,797 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 77e3e5908c8d586f7cad974ff9b02535 in 1243ms, sequenceid=233, compaction requested=true 2024-11-16T10:16:32,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:32,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:32,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-16T10:16:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-16T10:16:32,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-16T10:16:32,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7020 sec 2024-11-16T10:16:32,801 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.7060 sec 2024-11-16T10:16:33,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:33,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77e3e5908c8d586f7cad974ff9b02535 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-16T10:16:33,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=A 2024-11-16T10:16:33,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:33,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=B 2024-11-16T10:16:33,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:33,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=C 2024-11-16T10:16:33,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:33,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/edb86bd91dc244d9add5ec46f40f4056 is 50, key is test_row_0/A:col10/1731752191884/Put/seqid=0 2024-11-16T10:16:33,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742328_1504 (size=14541) 2024-11-16T10:16:33,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/edb86bd91dc244d9add5ec46f40f4056 2024-11-16T10:16:33,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/6e05488f341642608e87d9dd3e86b237 is 50, key is test_row_0/B:col10/1731752191884/Put/seqid=0 2024-11-16T10:16:33,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742329_1505 (size=12151) 2024-11-16T10:16:33,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752253072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:33,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:33,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752253081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:33,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:33,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752253082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:33,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:33,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:33,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752253194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:33,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:33,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752253194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:33,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:33,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:33,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752253204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:33,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T10:16:33,244 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:33,244 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:16:33,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-16T10:16:33,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T10:16:33,246 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:16:33,246 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:16:33,246 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:16:33,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T10:16:33,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T10:16:33,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:33,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:33,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:33,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:33,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:33,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:33,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:33,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752253414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:33,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:33,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752253424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:33,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:33,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752253424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:33,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:33,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:33,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/6e05488f341642608e87d9dd3e86b237 2024-11-16T10:16:33,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/f0965fe515eb4a04a135d00bd2bdb796 is 50, key is test_row_0/C:col10/1731752191884/Put/seqid=0 2024-11-16T10:16:33,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742330_1506 (size=12151) 2024-11-16T10:16:33,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T10:16:33,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:33,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:33,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:33,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:33,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:33,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T10:16:33,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T10:16:33,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:33,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:33,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:33,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:33,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:33,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:33,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:33,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752253734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:33,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:33,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752253744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:33,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:33,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752253744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:33,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:33,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:33,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:33,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T10:16:33,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:33,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:33,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:33,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:33,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:33,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:33,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T10:16:33,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/f0965fe515eb4a04a135d00bd2bdb796 2024-11-16T10:16:33,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/edb86bd91dc244d9add5ec46f40f4056 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/edb86bd91dc244d9add5ec46f40f4056 2024-11-16T10:16:33,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/edb86bd91dc244d9add5ec46f40f4056, entries=200, sequenceid=254, filesize=14.2 K 2024-11-16T10:16:33,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/6e05488f341642608e87d9dd3e86b237 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/6e05488f341642608e87d9dd3e86b237 2024-11-16T10:16:33,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/6e05488f341642608e87d9dd3e86b237, entries=150, sequenceid=254, filesize=11.9 K 2024-11-16T10:16:33,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/f0965fe515eb4a04a135d00bd2bdb796 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/f0965fe515eb4a04a135d00bd2bdb796 2024-11-16T10:16:33,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/f0965fe515eb4a04a135d00bd2bdb796, entries=150, sequenceid=254, filesize=11.9 K 2024-11-16T10:16:33,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 77e3e5908c8d586f7cad974ff9b02535 in 848ms, sequenceid=254, compaction requested=true 2024-11-16T10:16:33,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:33,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77e3e5908c8d586f7cad974ff9b02535:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:16:33,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:33,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77e3e5908c8d586f7cad974ff9b02535:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:16:33,898 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:16:33,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:33,898 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:16:33,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77e3e5908c8d586f7cad974ff9b02535:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:16:33,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:33,899 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49082 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:16:33,899 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:16:33,899 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 77e3e5908c8d586f7cad974ff9b02535/B is initiating minor compaction (all files) 2024-11-16T10:16:33,899 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 77e3e5908c8d586f7cad974ff9b02535/A is initiating minor compaction (all files) 2024-11-16T10:16:33,899 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77e3e5908c8d586f7cad974ff9b02535/B in TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:33,899 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77e3e5908c8d586f7cad974ff9b02535/A in TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:33,899 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/19eeb9a080044da8a53fd60b1e6907b1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/8f38e11689f54e1a8860bf4d3da7de31, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/9bb16961a2cd40a583536ec47eadf41c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/6e05488f341642608e87d9dd3e86b237] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp, totalSize=47.9 K 2024-11-16T10:16:33,899 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/c3dfc6f895bc46129dadfac15d05c062, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/a65d9a96502a427e958172f71298f934, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/90489a00731c496ca38cb0249cbb521b, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/edb86bd91dc244d9add5ec46f40f4056] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp, totalSize=52.6 K 2024-11-16T10:16:33,899 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3dfc6f895bc46129dadfac15d05c062, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731752189069 2024-11-16T10:16:33,899 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 19eeb9a080044da8a53fd60b1e6907b1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731752189069 2024-11-16T10:16:33,899 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting a65d9a96502a427e958172f71298f934, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731752189457 2024-11-16T10:16:33,899 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f38e11689f54e1a8860bf4d3da7de31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731752189457 2024-11-16T10:16:33,900 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90489a00731c496ca38cb0249cbb521b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731752190666 2024-11-16T10:16:33,900 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bb16961a2cd40a583536ec47eadf41c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731752190666 2024-11-16T10:16:33,900 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting edb86bd91dc244d9add5ec46f40f4056, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731752191877 2024-11-16T10:16:33,900 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e05488f341642608e87d9dd3e86b237, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731752191877 2024-11-16T10:16:33,905 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77e3e5908c8d586f7cad974ff9b02535#B#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:33,905 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77e3e5908c8d586f7cad974ff9b02535#A#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:33,905 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/8eb9c443496c412da55493e91be1708f is 50, key is test_row_0/B:col10/1731752191884/Put/seqid=0 2024-11-16T10:16:33,906 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/44ee9eacb6df427f8a6ddec8ad10821c is 50, key is test_row_0/A:col10/1731752191884/Put/seqid=0 2024-11-16T10:16:33,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742331_1507 (size=12765) 2024-11-16T10:16:33,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742332_1508 (size=12765) 2024-11-16T10:16:33,912 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/44ee9eacb6df427f8a6ddec8ad10821c as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/44ee9eacb6df427f8a6ddec8ad10821c 2024-11-16T10:16:33,915 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77e3e5908c8d586f7cad974ff9b02535/A of 77e3e5908c8d586f7cad974ff9b02535 into 44ee9eacb6df427f8a6ddec8ad10821c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:33,915 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:33,915 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., storeName=77e3e5908c8d586f7cad974ff9b02535/A, priority=12, startTime=1731752193898; duration=0sec 2024-11-16T10:16:33,915 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:33,915 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77e3e5908c8d586f7cad974ff9b02535:A 2024-11-16T10:16:33,915 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:16:33,916 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49082 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:16:33,916 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 77e3e5908c8d586f7cad974ff9b02535/C is initiating minor compaction (all files) 2024-11-16T10:16:33,916 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77e3e5908c8d586f7cad974ff9b02535/C in TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:33,916 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/09e7c93ff5a046bbaa900b59fa200c56, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/8bed6301eac548ee85c8e4f4ac0437bc, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/890fd37f70dc415292da867ee3388014, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/f0965fe515eb4a04a135d00bd2bdb796] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp, totalSize=47.9 K 2024-11-16T10:16:33,916 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09e7c93ff5a046bbaa900b59fa200c56, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731752189069 2024-11-16T10:16:33,917 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bed6301eac548ee85c8e4f4ac0437bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731752189457 2024-11-16T10:16:33,917 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 890fd37f70dc415292da867ee3388014, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731752190666 2024-11-16T10:16:33,917 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0965fe515eb4a04a135d00bd2bdb796, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731752191877 2024-11-16T10:16:33,926 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77e3e5908c8d586f7cad974ff9b02535#C#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:33,926 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/b0b07aee41be40c0a608cb2031d3c8ee is 50, key is test_row_0/C:col10/1731752191884/Put/seqid=0 2024-11-16T10:16:33,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742333_1509 (size=12765) 2024-11-16T10:16:34,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T10:16:34,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:34,009 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 77e3e5908c8d586f7cad974ff9b02535 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T10:16:34,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=A 2024-11-16T10:16:34,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:34,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=B 2024-11-16T10:16:34,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:34,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=C 2024-11-16T10:16:34,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:34,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/b329aa16252a43f6be1e196b3b7190e6 is 50, key is test_row_0/A:col10/1731752193071/Put/seqid=0 2024-11-16T10:16:34,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742334_1510 (size=12301) 2024-11-16T10:16:34,031 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/b329aa16252a43f6be1e196b3b7190e6 2024-11-16T10:16:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/053849a7b0214c59996cf2a87868a593 is 50, key is test_row_0/B:col10/1731752193071/Put/seqid=0 2024-11-16T10:16:34,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742335_1511 (size=12301) 2024-11-16T10:16:34,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:34,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:34,312 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/8eb9c443496c412da55493e91be1708f as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/8eb9c443496c412da55493e91be1708f 2024-11-16T10:16:34,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752194290, max timestamp=1731752194312Num uniques 859; 2024-11-16T10:16:34,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752194290, max timestamp=1731752194312Num uniques 859; 2024-11-16T10:16:34,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752194290, max timestamp=1731752194312Num uniques 859; 2024-11-16T10:16:34,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:16:34,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:16:34,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:16:34,318 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77e3e5908c8d586f7cad974ff9b02535/B of 77e3e5908c8d586f7cad974ff9b02535 into 8eb9c443496c412da55493e91be1708f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:34,318 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:34,318 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., storeName=77e3e5908c8d586f7cad974ff9b02535/B, priority=12, startTime=1731752193898; duration=0sec 2024-11-16T10:16:34,318 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:34,318 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77e3e5908c8d586f7cad974ff9b02535:B 2024-11-16T10:16:34,333 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/b0b07aee41be40c0a608cb2031d3c8ee as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/b0b07aee41be40c0a608cb2031d3c8ee 2024-11-16T10:16:34,339 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77e3e5908c8d586f7cad974ff9b02535/C of 77e3e5908c8d586f7cad974ff9b02535 into b0b07aee41be40c0a608cb2031d3c8ee(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:34,339 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:34,339 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., storeName=77e3e5908c8d586f7cad974ff9b02535/C, priority=12, startTime=1731752193898; duration=0sec 2024-11-16T10:16:34,339 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:34,339 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77e3e5908c8d586f7cad974ff9b02535:C 2024-11-16T10:16:34,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:34,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752254364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:34,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:34,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:34,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:34,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:34,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752254365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:34,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:34,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:34,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:34,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:34,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752254368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:34,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:34,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:34,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:34,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T10:16:34,439 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/053849a7b0214c59996cf2a87868a593 2024-11-16T10:16:34,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/08a402da80774697aadd0c8a589d671f is 50, key is test_row_0/C:col10/1731752193071/Put/seqid=0 2024-11-16T10:16:34,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742336_1512 (size=12301) 2024-11-16T10:16:34,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:34,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752254484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:34,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:34,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752254484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:34,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:34,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752254494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:34,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:34,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:34,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:34,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:34,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:34,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:34,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752254704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:34,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:34,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:34,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752254704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:34,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:34,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:34,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:34,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:34,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752254704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:34,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:34,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:34,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:34,850 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/08a402da80774697aadd0c8a589d671f 2024-11-16T10:16:34,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/b329aa16252a43f6be1e196b3b7190e6 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/b329aa16252a43f6be1e196b3b7190e6 2024-11-16T10:16:34,856 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/b329aa16252a43f6be1e196b3b7190e6, entries=150, sequenceid=266, filesize=12.0 K 2024-11-16T10:16:34,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/053849a7b0214c59996cf2a87868a593 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/053849a7b0214c59996cf2a87868a593 2024-11-16T10:16:34,859 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/053849a7b0214c59996cf2a87868a593, entries=150, sequenceid=266, filesize=12.0 K 2024-11-16T10:16:34,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/08a402da80774697aadd0c8a589d671f as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/08a402da80774697aadd0c8a589d671f 2024-11-16T10:16:34,864 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/08a402da80774697aadd0c8a589d671f, entries=150, sequenceid=266, filesize=12.0 K 2024-11-16T10:16:34,865 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 77e3e5908c8d586f7cad974ff9b02535 in 855ms, sequenceid=266, compaction requested=false 2024-11-16T10:16:34,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:34,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:34,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-16T10:16:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-16T10:16:34,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-16T10:16:34,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6200 sec 2024-11-16T10:16:34,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.6250 sec 2024-11-16T10:16:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:35,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77e3e5908c8d586f7cad974ff9b02535 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-16T10:16:35,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=A 2024-11-16T10:16:35,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:35,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=B 2024-11-16T10:16:35,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:35,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=C 2024-11-16T10:16:35,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:35,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/267f3863d22d462c96bf07ebfe4803bf is 50, key is test_row_0/A:col10/1731752194362/Put/seqid=0 2024-11-16T10:16:35,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742337_1513 (size=14741) 2024-11-16T10:16:35,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/267f3863d22d462c96bf07ebfe4803bf 2024-11-16T10:16:35,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/30048b0585914555b5b549a50ab1071f is 50, key is test_row_0/B:col10/1731752194362/Put/seqid=0 2024-11-16T10:16:35,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742338_1514 (size=12301) 2024-11-16T10:16:35,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752255043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:35,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:35,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:35,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752255047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:35,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:35,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752255048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:35,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:35,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752255154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:35,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:35,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:35,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752255163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:35,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752255174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:35,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:35,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752255374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:35,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:35,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752255383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T10:16:35,394 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:35,394 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:16:35,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-16T10:16:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T10:16:35,396 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:16:35,396 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:16:35,396 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:16:35,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752255394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:35,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/30048b0585914555b5b549a50ab1071f 2024-11-16T10:16:35,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/88b52efad98847fdab171f03f59654af is 50, key is test_row_0/C:col10/1731752194362/Put/seqid=0 2024-11-16T10:16:35,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742339_1515 (size=12301) 2024-11-16T10:16:35,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/88b52efad98847fdab171f03f59654af 2024-11-16T10:16:35,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/267f3863d22d462c96bf07ebfe4803bf as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/267f3863d22d462c96bf07ebfe4803bf 2024-11-16T10:16:35,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/267f3863d22d462c96bf07ebfe4803bf, entries=200, sequenceid=304, filesize=14.4 K 2024-11-16T10:16:35,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/30048b0585914555b5b549a50ab1071f as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/30048b0585914555b5b549a50ab1071f 2024-11-16T10:16:35,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/30048b0585914555b5b549a50ab1071f, entries=150, sequenceid=304, filesize=12.0 K 2024-11-16T10:16:35,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/88b52efad98847fdab171f03f59654af as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/88b52efad98847fdab171f03f59654af 2024-11-16T10:16:35,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/88b52efad98847fdab171f03f59654af, entries=150, sequenceid=304, filesize=12.0 K 2024-11-16T10:16:35,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 77e3e5908c8d586f7cad974ff9b02535 in 451ms, sequenceid=304, compaction requested=true 2024-11-16T10:16:35,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:35,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77e3e5908c8d586f7cad974ff9b02535:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:16:35,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:35,479 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:35,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77e3e5908c8d586f7cad974ff9b02535:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:16:35,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:35,479 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:35,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77e3e5908c8d586f7cad974ff9b02535:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:16:35,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:35,480 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:35,480 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 77e3e5908c8d586f7cad974ff9b02535/B is initiating minor compaction (all files) 2024-11-16T10:16:35,480 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77e3e5908c8d586f7cad974ff9b02535/B in TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:35,480 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/8eb9c443496c412da55493e91be1708f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/053849a7b0214c59996cf2a87868a593, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/30048b0585914555b5b549a50ab1071f] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp, totalSize=36.5 K 2024-11-16T10:16:35,480 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39807 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:35,480 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 77e3e5908c8d586f7cad974ff9b02535/A is initiating minor compaction (all files) 2024-11-16T10:16:35,480 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77e3e5908c8d586f7cad974ff9b02535/A in TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:35,480 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/44ee9eacb6df427f8a6ddec8ad10821c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/b329aa16252a43f6be1e196b3b7190e6, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/267f3863d22d462c96bf07ebfe4803bf] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp, totalSize=38.9 K 2024-11-16T10:16:35,480 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 8eb9c443496c412da55493e91be1708f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731752191877 2024-11-16T10:16:35,481 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44ee9eacb6df427f8a6ddec8ad10821c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731752191877 2024-11-16T10:16:35,481 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 053849a7b0214c59996cf2a87868a593, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731752193066 2024-11-16T10:16:35,481 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting b329aa16252a43f6be1e196b3b7190e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731752193066 2024-11-16T10:16:35,481 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 30048b0585914555b5b549a50ab1071f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731752194362 2024-11-16T10:16:35,481 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 267f3863d22d462c96bf07ebfe4803bf, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731752194362 2024-11-16T10:16:35,486 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77e3e5908c8d586f7cad974ff9b02535#B#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:35,486 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/07005e688c8e434cbfc5822720d389a1 is 50, key is test_row_0/B:col10/1731752194362/Put/seqid=0 2024-11-16T10:16:35,487 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77e3e5908c8d586f7cad974ff9b02535#A#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:35,488 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/f57df356d2cb42f098f7f5024b272f99 is 50, key is test_row_0/A:col10/1731752194362/Put/seqid=0 2024-11-16T10:16:35,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742341_1517 (size=13017) 2024-11-16T10:16:35,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742340_1516 (size=13017) 2024-11-16T10:16:35,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T10:16:35,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T10:16:35,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:35,548 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 77e3e5908c8d586f7cad974ff9b02535 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T10:16:35,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=A 2024-11-16T10:16:35,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:35,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=B 2024-11-16T10:16:35,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:35,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=C 2024-11-16T10:16:35,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:35,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/9483d07e6c954086917d4351b4b4004f is 50, key is test_row_0/A:col10/1731752195046/Put/seqid=0 2024-11-16T10:16:35,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742342_1518 (size=12301) 2024-11-16T10:16:35,559 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/9483d07e6c954086917d4351b4b4004f 2024-11-16T10:16:35,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/82011fbd11324f86a2b9c7c8c31f0dae is 50, key is test_row_0/B:col10/1731752195046/Put/seqid=0 2024-11-16T10:16:35,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742343_1519 (size=12301) 2024-11-16T10:16:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:35,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T10:16:35,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731752195735, max timestamp=1731752195758Num uniques 859; 2024-11-16T10:16:35,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T10:16:35,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731752195735, max timestamp=1731752195758Num uniques 859; 2024-11-16T10:16:35,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T10:16:35,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731752195735, max timestamp=1731752195758Num uniques 859; 2024-11-16T10:16:35,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T10:16:35,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752255811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:35,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752255812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:35,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752255825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:35,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:35,903 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/07005e688c8e434cbfc5822720d389a1 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/07005e688c8e434cbfc5822720d389a1 2024-11-16T10:16:35,903 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/f57df356d2cb42f098f7f5024b272f99 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/f57df356d2cb42f098f7f5024b272f99 2024-11-16T10:16:35,907 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77e3e5908c8d586f7cad974ff9b02535/B of 77e3e5908c8d586f7cad974ff9b02535 into 07005e688c8e434cbfc5822720d389a1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:35,907 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:35,907 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., storeName=77e3e5908c8d586f7cad974ff9b02535/B, priority=13, startTime=1731752195479; duration=0sec 2024-11-16T10:16:35,907 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:35,907 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77e3e5908c8d586f7cad974ff9b02535:B 2024-11-16T10:16:35,907 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:35,908 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77e3e5908c8d586f7cad974ff9b02535/A of 77e3e5908c8d586f7cad974ff9b02535 into f57df356d2cb42f098f7f5024b272f99(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:35,908 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:35,908 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., storeName=77e3e5908c8d586f7cad974ff9b02535/A, priority=13, startTime=1731752195479; duration=0sec 2024-11-16T10:16:35,908 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:35,908 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77e3e5908c8d586f7cad974ff9b02535:A 2024-11-16T10:16:35,908 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:35,909 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 77e3e5908c8d586f7cad974ff9b02535/C is initiating minor compaction (all files) 2024-11-16T10:16:35,909 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77e3e5908c8d586f7cad974ff9b02535/C in TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:35,909 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/b0b07aee41be40c0a608cb2031d3c8ee, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/08a402da80774697aadd0c8a589d671f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/88b52efad98847fdab171f03f59654af] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp, totalSize=36.5 K 2024-11-16T10:16:35,909 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting b0b07aee41be40c0a608cb2031d3c8ee, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731752191877 2024-11-16T10:16:35,909 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 08a402da80774697aadd0c8a589d671f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731752193066 2024-11-16T10:16:35,909 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 88b52efad98847fdab171f03f59654af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731752194362 2024-11-16T10:16:35,915 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77e3e5908c8d586f7cad974ff9b02535#C#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:35,916 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/158a082610e04cf2878cd173e971d3c9 is 50, key is test_row_0/C:col10/1731752194362/Put/seqid=0 2024-11-16T10:16:35,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742344_1520 (size=13017) 2024-11-16T10:16:35,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752255935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:35,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:35,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752255937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:35,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:35,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:35,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752255944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:35,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:35,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:35,969 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/82011fbd11324f86a2b9c7c8c31f0dae 2024-11-16T10:16:35,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/aab01e280c5e47a3acf69586e30a4f30 is 50, key is test_row_0/C:col10/1731752195046/Put/seqid=0 2024-11-16T10:16:35,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742345_1521 (size=12301) 2024-11-16T10:16:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T10:16:36,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:36,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752256144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:36,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752256144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:36,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:36,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752256164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:36,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:36,327 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/158a082610e04cf2878cd173e971d3c9 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/158a082610e04cf2878cd173e971d3c9 2024-11-16T10:16:36,330 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77e3e5908c8d586f7cad974ff9b02535/C of 77e3e5908c8d586f7cad974ff9b02535 into 158a082610e04cf2878cd173e971d3c9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:36,330 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:36,330 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., storeName=77e3e5908c8d586f7cad974ff9b02535/C, priority=13, startTime=1731752195479; duration=0sec 2024-11-16T10:16:36,330 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:36,330 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77e3e5908c8d586f7cad974ff9b02535:C 2024-11-16T10:16:36,380 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/aab01e280c5e47a3acf69586e30a4f30 2024-11-16T10:16:36,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/9483d07e6c954086917d4351b4b4004f as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/9483d07e6c954086917d4351b4b4004f 2024-11-16T10:16:36,386 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/9483d07e6c954086917d4351b4b4004f, entries=150, sequenceid=314, filesize=12.0 K 2024-11-16T10:16:36,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/82011fbd11324f86a2b9c7c8c31f0dae as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/82011fbd11324f86a2b9c7c8c31f0dae 2024-11-16T10:16:36,389 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/82011fbd11324f86a2b9c7c8c31f0dae, entries=150, sequenceid=314, filesize=12.0 K 2024-11-16T10:16:36,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/aab01e280c5e47a3acf69586e30a4f30 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/aab01e280c5e47a3acf69586e30a4f30 2024-11-16T10:16:36,393 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/aab01e280c5e47a3acf69586e30a4f30, entries=150, sequenceid=314, filesize=12.0 K 2024-11-16T10:16:36,393 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 77e3e5908c8d586f7cad974ff9b02535 in 845ms, sequenceid=314, compaction requested=false 2024-11-16T10:16:36,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:36,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:36,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-16T10:16:36,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-16T10:16:36,396 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-16T10:16:36,396 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 998 msec 2024-11-16T10:16:36,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.0020 sec 2024-11-16T10:16:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:36,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77e3e5908c8d586f7cad974ff9b02535 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-16T10:16:36,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=A 2024-11-16T10:16:36,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:36,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=B 2024-11-16T10:16:36,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:36,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=C 2024-11-16T10:16:36,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:36,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/35a2aaea7bff47a58bdae065a273dfca is 50, key is test_row_0/A:col10/1731752195791/Put/seqid=0 2024-11-16T10:16:36,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742346_1522 (size=14741) 2024-11-16T10:16:36,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752256489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:36,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:36,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752256493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:36,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:36,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:36,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752256497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:36,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T10:16:36,534 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:36,534 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:16:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-16T10:16:36,536 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:16:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T10:16:36,536 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:16:36,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:16:36,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752256614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:36,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:36,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752256614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:36,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:36,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752256614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T10:16:36,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T10:16:36,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:36,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:36,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:36,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:36,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:36,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T10:16:36,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:36,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:36,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:36,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:36,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:36,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752256834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:36,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:36,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752256834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:36,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:36,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:36,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752256844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:36,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:36,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:36,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T10:16:36,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/35a2aaea7bff47a58bdae065a273dfca 2024-11-16T10:16:36,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/36f55a7333da4b6dbf406157d60b1912 is 50, key is test_row_0/B:col10/1731752195791/Put/seqid=0 2024-11-16T10:16:36,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742347_1523 (size=12301) 2024-11-16T10:16:36,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T10:16:36,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:36,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:36,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:36,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:36,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:36,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:37,028 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T10:16:37,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T10:16:37,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:37,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:37,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:37,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:37,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:37,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752257154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:37,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752257154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:37,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752257154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T10:16:37,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T10:16:37,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:37,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:37,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:37,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:37,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:37,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/36f55a7333da4b6dbf406157d60b1912 2024-11-16T10:16:37,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/e5927dcbf77b4c0c8ed21e48763b5be2 is 50, key is test_row_0/C:col10/1731752195791/Put/seqid=0 2024-11-16T10:16:37,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742348_1524 (size=12301) 2024-11-16T10:16:37,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T10:16:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:37,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:37,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:37,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T10:16:37,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:37,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:37,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:37,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:37,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:37,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:37,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:37,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51770 deadline: 1731752257664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:37,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:37,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51778 deadline: 1731752257664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T10:16:37,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51802 deadline: 1731752257674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., hostname=b16ff38458c1,45417,1731752048912, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:16:37,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/e5927dcbf77b4c0c8ed21e48763b5be2 2024-11-16T10:16:37,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/35a2aaea7bff47a58bdae065a273dfca as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/35a2aaea7bff47a58bdae065a273dfca 2024-11-16T10:16:37,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/35a2aaea7bff47a58bdae065a273dfca, entries=200, sequenceid=353, filesize=14.4 K 2024-11-16T10:16:37,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/36f55a7333da4b6dbf406157d60b1912 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/36f55a7333da4b6dbf406157d60b1912 2024-11-16T10:16:37,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/36f55a7333da4b6dbf406157d60b1912, entries=150, sequenceid=353, filesize=12.0 K 2024-11-16T10:16:37,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/e5927dcbf77b4c0c8ed21e48763b5be2 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/e5927dcbf77b4c0c8ed21e48763b5be2 2024-11-16T10:16:37,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/e5927dcbf77b4c0c8ed21e48763b5be2, entries=150, sequenceid=353, filesize=12.0 K 2024-11-16T10:16:37,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 77e3e5908c8d586f7cad974ff9b02535 in 1254ms, sequenceid=353, compaction requested=true 2024-11-16T10:16:37,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:37,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77e3e5908c8d586f7cad974ff9b02535:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:16:37,725 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:37,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:37,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77e3e5908c8d586f7cad974ff9b02535:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:16:37,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:37,725 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:37,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77e3e5908c8d586f7cad974ff9b02535:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:16:37,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:37,725 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:37,725 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 77e3e5908c8d586f7cad974ff9b02535/A is initiating minor compaction (all files) 2024-11-16T10:16:37,725 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:37,725 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77e3e5908c8d586f7cad974ff9b02535/A in TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:37,725 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 77e3e5908c8d586f7cad974ff9b02535/B is initiating minor compaction (all files) 2024-11-16T10:16:37,725 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77e3e5908c8d586f7cad974ff9b02535/B in TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:37,725 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/f57df356d2cb42f098f7f5024b272f99, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/9483d07e6c954086917d4351b4b4004f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/35a2aaea7bff47a58bdae065a273dfca] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp, totalSize=39.1 K 2024-11-16T10:16:37,725 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/07005e688c8e434cbfc5822720d389a1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/82011fbd11324f86a2b9c7c8c31f0dae, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/36f55a7333da4b6dbf406157d60b1912] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp, totalSize=36.7 K 2024-11-16T10:16:37,726 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 07005e688c8e434cbfc5822720d389a1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731752194362 2024-11-16T10:16:37,726 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting f57df356d2cb42f098f7f5024b272f99, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731752194362 2024-11-16T10:16:37,726 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 82011fbd11324f86a2b9c7c8c31f0dae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731752195041 2024-11-16T10:16:37,726 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9483d07e6c954086917d4351b4b4004f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731752195041 2024-11-16T10:16:37,726 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 36f55a7333da4b6dbf406157d60b1912, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731752195791 2024-11-16T10:16:37,726 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35a2aaea7bff47a58bdae065a273dfca, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731752195791 2024-11-16T10:16:37,732 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77e3e5908c8d586f7cad974ff9b02535#A#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:37,732 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/d68642d581a24fe182a8cd274c6adc13 is 50, key is test_row_0/A:col10/1731752195791/Put/seqid=0 2024-11-16T10:16:37,736 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77e3e5908c8d586f7cad974ff9b02535#B#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:37,737 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/7a5d85b3769e4193aaac32096bb41377 is 50, key is test_row_0/B:col10/1731752195791/Put/seqid=0 2024-11-16T10:16:37,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742349_1525 (size=13119) 2024-11-16T10:16:37,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742350_1526 (size=13119) 2024-11-16T10:16:37,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T10:16:37,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:37,761 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 77e3e5908c8d586f7cad974ff9b02535 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-16T10:16:37,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=A 2024-11-16T10:16:37,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:37,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=B 2024-11-16T10:16:37,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:37,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=C 2024-11-16T10:16:37,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:37,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/baab6bfcde0c4feb9f644487a99d6eab is 50, key is test_row_0/A:col10/1731752196496/Put/seqid=0 2024-11-16T10:16:37,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742351_1527 (size=12301) 2024-11-16T10:16:38,143 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/d68642d581a24fe182a8cd274c6adc13 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/d68642d581a24fe182a8cd274c6adc13 2024-11-16T10:16:38,143 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/7a5d85b3769e4193aaac32096bb41377 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/7a5d85b3769e4193aaac32096bb41377 2024-11-16T10:16:38,147 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77e3e5908c8d586f7cad974ff9b02535/B of 77e3e5908c8d586f7cad974ff9b02535 into 7a5d85b3769e4193aaac32096bb41377(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:38,147 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77e3e5908c8d586f7cad974ff9b02535/A of 77e3e5908c8d586f7cad974ff9b02535 into d68642d581a24fe182a8cd274c6adc13(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:38,147 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:38,147 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:38,147 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., storeName=77e3e5908c8d586f7cad974ff9b02535/B, priority=13, startTime=1731752197725; duration=0sec 2024-11-16T10:16:38,147 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., storeName=77e3e5908c8d586f7cad974ff9b02535/A, priority=13, startTime=1731752197724; duration=0sec 2024-11-16T10:16:38,147 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:38,147 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77e3e5908c8d586f7cad974ff9b02535:B 2024-11-16T10:16:38,147 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:38,147 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77e3e5908c8d586f7cad974ff9b02535:A 2024-11-16T10:16:38,147 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:38,148 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:38,148 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 77e3e5908c8d586f7cad974ff9b02535/C is initiating minor compaction (all files) 2024-11-16T10:16:38,148 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77e3e5908c8d586f7cad974ff9b02535/C in TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:38,148 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/158a082610e04cf2878cd173e971d3c9, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/aab01e280c5e47a3acf69586e30a4f30, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/e5927dcbf77b4c0c8ed21e48763b5be2] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp, totalSize=36.7 K 2024-11-16T10:16:38,148 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 158a082610e04cf2878cd173e971d3c9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731752194362 2024-11-16T10:16:38,149 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting aab01e280c5e47a3acf69586e30a4f30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731752195041 2024-11-16T10:16:38,149 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting e5927dcbf77b4c0c8ed21e48763b5be2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731752195791 2024-11-16T10:16:38,154 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77e3e5908c8d586f7cad974ff9b02535#C#compaction#453 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:38,155 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/65c828e3cd2f461b9b9a6259142e72d6 is 50, key is test_row_0/C:col10/1731752195791/Put/seqid=0 2024-11-16T10:16:38,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742352_1528 (size=13119) 2024-11-16T10:16:38,175 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/baab6bfcde0c4feb9f644487a99d6eab 2024-11-16T10:16:38,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/8b3b4b50e60144948517b81445c8915d is 50, key is test_row_0/B:col10/1731752196496/Put/seqid=0 2024-11-16T10:16:38,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742353_1529 (size=12301) 2024-11-16T10:16:38,461 INFO [Thread-2039 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2039. 2024-11-16T10:16:38,461 DEBUG [Thread-2039 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:16:38,461 DEBUG [Thread-2039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:38,461 DEBUG [Thread-2039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:38,461 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:16:38,463 INFO [Thread-2043 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2043. 2024-11-16T10:16:38,463 DEBUG [Thread-2043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:16:38,463 DEBUG [Thread-2043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:38,463 DEBUG [Thread-2043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:38,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:16:38,465 INFO [Thread-2045 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2045. 2024-11-16T10:16:38,465 DEBUG [Thread-2045 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:16:38,465 DEBUG [Thread-2045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:38,465 DEBUG [Thread-2045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:38,465 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:16:38,468 INFO [Thread-2041 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2041. 2024-11-16T10:16:38,468 DEBUG [Thread-2041 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:16:38,468 DEBUG [Thread-2041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:38,468 DEBUG [Thread-2041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:38,468 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:16:38,469 INFO [Thread-2037 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2037. 2024-11-16T10:16:38,469 DEBUG [Thread-2037 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:16:38,469 DEBUG [Thread-2037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:38,469 DEBUG [Thread-2037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:38,469 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:16:38,568 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/65c828e3cd2f461b9b9a6259142e72d6 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/65c828e3cd2f461b9b9a6259142e72d6 2024-11-16T10:16:38,574 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77e3e5908c8d586f7cad974ff9b02535/C of 77e3e5908c8d586f7cad974ff9b02535 into 65c828e3cd2f461b9b9a6259142e72d6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:38,574 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:38,574 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., storeName=77e3e5908c8d586f7cad974ff9b02535/C, priority=13, startTime=1731752197725; duration=0sec 2024-11-16T10:16:38,574 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:38,574 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77e3e5908c8d586f7cad974ff9b02535:C 2024-11-16T10:16:38,592 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/8b3b4b50e60144948517b81445c8915d 2024-11-16T10:16:38,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/6db90bada4ef48b29053ea18fdbd7a23 is 50, key is test_row_0/C:col10/1731752196496/Put/seqid=0 2024-11-16T10:16:38,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742354_1530 (size=12301) 2024-11-16T10:16:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:38,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. as already flushing 2024-11-16T10:16:38,690 INFO [Thread-2032 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2032. 2024-11-16T10:16:38,690 INFO [Thread-2028 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2028. 2024-11-16T10:16:38,690 DEBUG [Thread-2032 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:16:38,690 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:38,690 DEBUG [Thread-2028 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:16:38,690 DEBUG [Thread-2028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:38,690 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:38,690 DEBUG [Thread-2028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:38,690 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:16:38,691 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:16:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T10:16:38,696 INFO [Thread-2026 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2026. 2024-11-16T10:16:38,696 DEBUG [Thread-2026 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:16:38,696 DEBUG [Thread-2026 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:38,696 DEBUG [Thread-2026 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:38,696 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:16:39,008 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/6db90bada4ef48b29053ea18fdbd7a23 2024-11-16T10:16:39,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/baab6bfcde0c4feb9f644487a99d6eab as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/baab6bfcde0c4feb9f644487a99d6eab 2024-11-16T10:16:39,022 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/baab6bfcde0c4feb9f644487a99d6eab, entries=150, sequenceid=362, filesize=12.0 K 2024-11-16T10:16:39,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/8b3b4b50e60144948517b81445c8915d as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/8b3b4b50e60144948517b81445c8915d 2024-11-16T10:16:39,027 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/8b3b4b50e60144948517b81445c8915d, entries=150, sequenceid=362, filesize=12.0 K 2024-11-16T10:16:39,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/6db90bada4ef48b29053ea18fdbd7a23 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/6db90bada4ef48b29053ea18fdbd7a23 2024-11-16T10:16:39,032 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/6db90bada4ef48b29053ea18fdbd7a23, entries=150, sequenceid=362, filesize=12.0 K 2024-11-16T10:16:39,032 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=20.13 KB/20610 for 77e3e5908c8d586f7cad974ff9b02535 in 1271ms, sequenceid=362, compaction requested=false 2024-11-16T10:16:39,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 77e3e5908c8d586f7cad974ff9b02535: 2024-11-16T10:16:39,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:39,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-16T10:16:39,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-16T10:16:39,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-16T10:16:39,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4980 sec 2024-11-16T10:16:39,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.5020 sec 2024-11-16T10:16:40,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T10:16:40,714 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:41,389 INFO [Thread-2034 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2034. 2024-11-16T10:16:41,389 DEBUG [Thread-2034 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:16:41,390 DEBUG [Thread-2034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:41,390 DEBUG [Thread-2034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:41,390 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:16:41,406 INFO [Thread-2030 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2030. 2024-11-16T10:16:41,406 DEBUG [Thread-2030 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:16:41,406 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:41,406 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:41,406 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:16:41,407 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T10:16:41,407 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-16T10:16:41,407 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-16T10:16:41,407 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-16T10:16:41,407 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-16T10:16:41,407 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-16T10:16:41,407 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T10:16:41,407 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T10:16:41,407 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1157 2024-11-16T10:16:41,407 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3465 rows 2024-11-16T10:16:41,407 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1154 2024-11-16T10:16:41,407 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3459 rows 2024-11-16T10:16:41,407 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1157 2024-11-16T10:16:41,407 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3471 rows 2024-11-16T10:16:41,407 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1157 2024-11-16T10:16:41,407 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3465 rows 2024-11-16T10:16:41,407 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1163 2024-11-16T10:16:41,407 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3480 rows 2024-11-16T10:16:41,407 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T10:16:41,407 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T10:16:41,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:41,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:41,408 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:16:41,409 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T10:16:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T10:16:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T10:16:41,414 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752201413"}]},"ts":"1731752201413"} 2024-11-16T10:16:41,416 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T10:16:41,416 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T10:16:41,417 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T10:16:41,420 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77e3e5908c8d586f7cad974ff9b02535, UNASSIGN}] 2024-11-16T10:16:41,421 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77e3e5908c8d586f7cad974ff9b02535, UNASSIGN 2024-11-16T10:16:41,422 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=77e3e5908c8d586f7cad974ff9b02535, regionState=CLOSING, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:41,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=77e3e5908c8d586f7cad974ff9b02535, UNASSIGN because future has completed 2024-11-16T10:16:41,424 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T10:16:41,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912}] 2024-11-16T10:16:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T10:16:41,577 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:41,577 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T10:16:41,577 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 77e3e5908c8d586f7cad974ff9b02535, disabling compactions & flushes 2024-11-16T10:16:41,577 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:41,577 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:41,577 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. after waiting 0 ms 2024-11-16T10:16:41,577 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:41,577 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing 77e3e5908c8d586f7cad974ff9b02535 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-16T10:16:41,577 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=A 2024-11-16T10:16:41,578 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:41,578 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=B 2024-11-16T10:16:41,578 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:41,578 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77e3e5908c8d586f7cad974ff9b02535, store=C 2024-11-16T10:16:41,578 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:41,583 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/494c0b7d75a94d37b22327374c5c5b9e is 50, key is test_row_0/A:col10/1731752201404/Put/seqid=0 2024-11-16T10:16:41,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742355_1531 (size=9857) 2024-11-16T10:16:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T10:16:41,988 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/494c0b7d75a94d37b22327374c5c5b9e 2024-11-16T10:16:41,997 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/8764c382e7d94a468218558067e74843 is 50, key is test_row_0/B:col10/1731752201404/Put/seqid=0 2024-11-16T10:16:42,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742356_1532 (size=9857) 2024-11-16T10:16:42,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T10:16:42,403 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/8764c382e7d94a468218558067e74843 2024-11-16T10:16:42,410 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/6f1aeb59da1145ccaf45e7726f8bfc9d is 50, key is test_row_0/C:col10/1731752201404/Put/seqid=0 2024-11-16T10:16:42,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742357_1533 (size=9857) 2024-11-16T10:16:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T10:16:42,814 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/6f1aeb59da1145ccaf45e7726f8bfc9d 2024-11-16T10:16:42,819 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/A/494c0b7d75a94d37b22327374c5c5b9e as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/494c0b7d75a94d37b22327374c5c5b9e 2024-11-16T10:16:42,822 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/494c0b7d75a94d37b22327374c5c5b9e, entries=100, sequenceid=373, filesize=9.6 K 2024-11-16T10:16:42,823 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/B/8764c382e7d94a468218558067e74843 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/8764c382e7d94a468218558067e74843 2024-11-16T10:16:42,826 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/8764c382e7d94a468218558067e74843, entries=100, sequenceid=373, filesize=9.6 K 2024-11-16T10:16:42,827 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/.tmp/C/6f1aeb59da1145ccaf45e7726f8bfc9d as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/6f1aeb59da1145ccaf45e7726f8bfc9d 2024-11-16T10:16:42,830 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/6f1aeb59da1145ccaf45e7726f8bfc9d, entries=100, sequenceid=373, filesize=9.6 K 2024-11-16T10:16:42,831 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 77e3e5908c8d586f7cad974ff9b02535 in 1254ms, sequenceid=373, compaction requested=true 2024-11-16T10:16:42,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/a962b08f2960446b96029c08580cc78c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/0b9325598db840039fbd396828c111b0, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/28bb02a02d6a42f3b73601b5eb7b408f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/f54f9d722a98451a96f5a9448d74b2ec, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/2c831c8721424193b9098c21e1702d26, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/1b68a554aa8e44239ad59e93e4ca2f6c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/54435b39ec79438aad50d72494957363, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/4e59aaa3369048d194dfdf1337ef2a08, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/887a2fd19e4c4773ac5b2f4ce80774af, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/1f9175b7ae794de0aa53a4453956e7dc, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/4fbd43942c4f4072a97d07f34af5df2f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/6f54ad978bcc475b8416e4ffd86b997c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/24df0799caae43138d9abe31ea1ec61f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/03da6bdd0c7a49799da88f2a48916d56, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/c3dfc6f895bc46129dadfac15d05c062, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/a65d9a96502a427e958172f71298f934, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/90489a00731c496ca38cb0249cbb521b, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/edb86bd91dc244d9add5ec46f40f4056, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/44ee9eacb6df427f8a6ddec8ad10821c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/b329aa16252a43f6be1e196b3b7190e6, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/267f3863d22d462c96bf07ebfe4803bf, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/f57df356d2cb42f098f7f5024b272f99, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/9483d07e6c954086917d4351b4b4004f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/35a2aaea7bff47a58bdae065a273dfca] to archive 2024-11-16T10:16:42,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:16:42,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/a962b08f2960446b96029c08580cc78c to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/a962b08f2960446b96029c08580cc78c 2024-11-16T10:16:42,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/0b9325598db840039fbd396828c111b0 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/0b9325598db840039fbd396828c111b0 2024-11-16T10:16:42,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/28bb02a02d6a42f3b73601b5eb7b408f to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/28bb02a02d6a42f3b73601b5eb7b408f 2024-11-16T10:16:42,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/f54f9d722a98451a96f5a9448d74b2ec to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/f54f9d722a98451a96f5a9448d74b2ec 2024-11-16T10:16:42,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/2c831c8721424193b9098c21e1702d26 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/2c831c8721424193b9098c21e1702d26 2024-11-16T10:16:42,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/1b68a554aa8e44239ad59e93e4ca2f6c to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/1b68a554aa8e44239ad59e93e4ca2f6c 2024-11-16T10:16:42,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/54435b39ec79438aad50d72494957363 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/54435b39ec79438aad50d72494957363 2024-11-16T10:16:42,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/4e59aaa3369048d194dfdf1337ef2a08 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/4e59aaa3369048d194dfdf1337ef2a08 2024-11-16T10:16:42,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/887a2fd19e4c4773ac5b2f4ce80774af to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/887a2fd19e4c4773ac5b2f4ce80774af 2024-11-16T10:16:42,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/1f9175b7ae794de0aa53a4453956e7dc to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/1f9175b7ae794de0aa53a4453956e7dc 2024-11-16T10:16:42,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/4fbd43942c4f4072a97d07f34af5df2f to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/4fbd43942c4f4072a97d07f34af5df2f 2024-11-16T10:16:42,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/6f54ad978bcc475b8416e4ffd86b997c to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/6f54ad978bcc475b8416e4ffd86b997c 2024-11-16T10:16:42,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/24df0799caae43138d9abe31ea1ec61f to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/24df0799caae43138d9abe31ea1ec61f 2024-11-16T10:16:42,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/03da6bdd0c7a49799da88f2a48916d56 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/03da6bdd0c7a49799da88f2a48916d56 2024-11-16T10:16:42,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/c3dfc6f895bc46129dadfac15d05c062 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/c3dfc6f895bc46129dadfac15d05c062 2024-11-16T10:16:42,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/a65d9a96502a427e958172f71298f934 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/a65d9a96502a427e958172f71298f934 2024-11-16T10:16:42,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/90489a00731c496ca38cb0249cbb521b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/90489a00731c496ca38cb0249cbb521b 2024-11-16T10:16:42,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/edb86bd91dc244d9add5ec46f40f4056 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/edb86bd91dc244d9add5ec46f40f4056 2024-11-16T10:16:42,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/44ee9eacb6df427f8a6ddec8ad10821c to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/44ee9eacb6df427f8a6ddec8ad10821c 2024-11-16T10:16:42,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/b329aa16252a43f6be1e196b3b7190e6 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/b329aa16252a43f6be1e196b3b7190e6 2024-11-16T10:16:42,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/267f3863d22d462c96bf07ebfe4803bf to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/267f3863d22d462c96bf07ebfe4803bf 2024-11-16T10:16:42,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/f57df356d2cb42f098f7f5024b272f99 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/f57df356d2cb42f098f7f5024b272f99 2024-11-16T10:16:42,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/9483d07e6c954086917d4351b4b4004f to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/9483d07e6c954086917d4351b4b4004f 2024-11-16T10:16:42,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/35a2aaea7bff47a58bdae065a273dfca to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/35a2aaea7bff47a58bdae065a273dfca 2024-11-16T10:16:42,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/3da868d564fc46f582b12eac759a290f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/a035f46ac57d4f658ef1b662998ecc84, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/c32445e4762a4bb5a6a3fc991ac567ed, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/2376d5e22c9f48ddb0f6749adf728ec8, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/e5c1918b1e66402181b601e1158bbe88, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/9da18e2f7d0f45828df1e6d5bbf8f4a0, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/28348b7200714fad87cad418c32ab69b, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/34683eb08b844576b36e8e7c6a9cca51, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/e77c51f34c504010b93189faa04539ed, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/98956f8d67f9475f9fb3572b3e1b41e1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/982cfa83804044f28ba1a64b1765a897, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/5466b95fa57a4cb68a72c9c5005ac9e5, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/c2677db0d82c43218f6a8934e7d37049, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/19eeb9a080044da8a53fd60b1e6907b1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/9a22d625e27546b0993c89e00011e82b, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/8f38e11689f54e1a8860bf4d3da7de31, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/9bb16961a2cd40a583536ec47eadf41c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/8eb9c443496c412da55493e91be1708f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/6e05488f341642608e87d9dd3e86b237, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/053849a7b0214c59996cf2a87868a593, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/07005e688c8e434cbfc5822720d389a1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/30048b0585914555b5b549a50ab1071f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/82011fbd11324f86a2b9c7c8c31f0dae, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/36f55a7333da4b6dbf406157d60b1912] to archive 2024-11-16T10:16:42,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:16:42,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/3da868d564fc46f582b12eac759a290f to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/3da868d564fc46f582b12eac759a290f 2024-11-16T10:16:42,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/a035f46ac57d4f658ef1b662998ecc84 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/a035f46ac57d4f658ef1b662998ecc84 2024-11-16T10:16:42,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/c32445e4762a4bb5a6a3fc991ac567ed to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/c32445e4762a4bb5a6a3fc991ac567ed 2024-11-16T10:16:42,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/2376d5e22c9f48ddb0f6749adf728ec8 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/2376d5e22c9f48ddb0f6749adf728ec8 2024-11-16T10:16:42,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/e5c1918b1e66402181b601e1158bbe88 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/e5c1918b1e66402181b601e1158bbe88 2024-11-16T10:16:42,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/9da18e2f7d0f45828df1e6d5bbf8f4a0 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/9da18e2f7d0f45828df1e6d5bbf8f4a0 2024-11-16T10:16:42,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/28348b7200714fad87cad418c32ab69b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/28348b7200714fad87cad418c32ab69b 2024-11-16T10:16:42,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/34683eb08b844576b36e8e7c6a9cca51 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/34683eb08b844576b36e8e7c6a9cca51 2024-11-16T10:16:42,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/e77c51f34c504010b93189faa04539ed to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/e77c51f34c504010b93189faa04539ed 2024-11-16T10:16:42,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/98956f8d67f9475f9fb3572b3e1b41e1 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/98956f8d67f9475f9fb3572b3e1b41e1 2024-11-16T10:16:42,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/982cfa83804044f28ba1a64b1765a897 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/982cfa83804044f28ba1a64b1765a897 2024-11-16T10:16:42,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/5466b95fa57a4cb68a72c9c5005ac9e5 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/5466b95fa57a4cb68a72c9c5005ac9e5 2024-11-16T10:16:42,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/c2677db0d82c43218f6a8934e7d37049 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/c2677db0d82c43218f6a8934e7d37049 2024-11-16T10:16:42,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/19eeb9a080044da8a53fd60b1e6907b1 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/19eeb9a080044da8a53fd60b1e6907b1 2024-11-16T10:16:42,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/9a22d625e27546b0993c89e00011e82b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/9a22d625e27546b0993c89e00011e82b 2024-11-16T10:16:42,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/8f38e11689f54e1a8860bf4d3da7de31 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/8f38e11689f54e1a8860bf4d3da7de31 2024-11-16T10:16:42,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/9bb16961a2cd40a583536ec47eadf41c to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/9bb16961a2cd40a583536ec47eadf41c 2024-11-16T10:16:42,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/8eb9c443496c412da55493e91be1708f to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/8eb9c443496c412da55493e91be1708f 2024-11-16T10:16:42,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/6e05488f341642608e87d9dd3e86b237 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/6e05488f341642608e87d9dd3e86b237 2024-11-16T10:16:42,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/053849a7b0214c59996cf2a87868a593 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/053849a7b0214c59996cf2a87868a593 2024-11-16T10:16:42,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/07005e688c8e434cbfc5822720d389a1 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/07005e688c8e434cbfc5822720d389a1 2024-11-16T10:16:42,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/30048b0585914555b5b549a50ab1071f to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/30048b0585914555b5b549a50ab1071f 2024-11-16T10:16:42,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/82011fbd11324f86a2b9c7c8c31f0dae to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/82011fbd11324f86a2b9c7c8c31f0dae 2024-11-16T10:16:42,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/36f55a7333da4b6dbf406157d60b1912 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/36f55a7333da4b6dbf406157d60b1912 2024-11-16T10:16:42,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/6716fc154e024be489d03e0d7fb83114, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/c205c7501bd94a8081a6be8d8cb0dfd4, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/93ffddd471984f0cad08134b05f37500, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/240c2f3040664c1c8d9b8c16f9d22d83, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/f7e294aa632a4d8b8cf313fe977d1112, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/489f8f6ad0e14b219bf30a6b931585a6, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/5e04f8849bb24646a3cbb4080792e154, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/aa9d4691e8ab44afbdc2b957fb928ec1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/ae32e20122bc44dd8793b65d5ed4c8b4, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/344516967b9748e6a991cc56480e76ab, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/e24999964eb34cb99ea91e9d978a8f13, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/90bad977472b49ba9d1a6932a4e302c9, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/29881b5c0dba40f59b88467cb9fc7b2d, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/09e7c93ff5a046bbaa900b59fa200c56, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/d479a914703741a3ad874288d127ff71, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/8bed6301eac548ee85c8e4f4ac0437bc, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/890fd37f70dc415292da867ee3388014, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/b0b07aee41be40c0a608cb2031d3c8ee, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/f0965fe515eb4a04a135d00bd2bdb796, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/08a402da80774697aadd0c8a589d671f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/158a082610e04cf2878cd173e971d3c9, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/88b52efad98847fdab171f03f59654af, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/aab01e280c5e47a3acf69586e30a4f30, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/e5927dcbf77b4c0c8ed21e48763b5be2] to archive 2024-11-16T10:16:42,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:16:42,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/6716fc154e024be489d03e0d7fb83114 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/6716fc154e024be489d03e0d7fb83114 2024-11-16T10:16:42,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/c205c7501bd94a8081a6be8d8cb0dfd4 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/c205c7501bd94a8081a6be8d8cb0dfd4 2024-11-16T10:16:42,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/93ffddd471984f0cad08134b05f37500 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/93ffddd471984f0cad08134b05f37500 2024-11-16T10:16:42,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/240c2f3040664c1c8d9b8c16f9d22d83 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/240c2f3040664c1c8d9b8c16f9d22d83 2024-11-16T10:16:42,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/f7e294aa632a4d8b8cf313fe977d1112 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/f7e294aa632a4d8b8cf313fe977d1112 2024-11-16T10:16:42,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/489f8f6ad0e14b219bf30a6b931585a6 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/489f8f6ad0e14b219bf30a6b931585a6 2024-11-16T10:16:42,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/5e04f8849bb24646a3cbb4080792e154 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/5e04f8849bb24646a3cbb4080792e154 2024-11-16T10:16:42,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/aa9d4691e8ab44afbdc2b957fb928ec1 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/aa9d4691e8ab44afbdc2b957fb928ec1 2024-11-16T10:16:42,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/ae32e20122bc44dd8793b65d5ed4c8b4 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/ae32e20122bc44dd8793b65d5ed4c8b4 2024-11-16T10:16:42,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/344516967b9748e6a991cc56480e76ab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/344516967b9748e6a991cc56480e76ab 2024-11-16T10:16:42,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/e24999964eb34cb99ea91e9d978a8f13 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/e24999964eb34cb99ea91e9d978a8f13 2024-11-16T10:16:42,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/90bad977472b49ba9d1a6932a4e302c9 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/90bad977472b49ba9d1a6932a4e302c9 2024-11-16T10:16:42,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/29881b5c0dba40f59b88467cb9fc7b2d to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/29881b5c0dba40f59b88467cb9fc7b2d 2024-11-16T10:16:42,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/09e7c93ff5a046bbaa900b59fa200c56 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/09e7c93ff5a046bbaa900b59fa200c56 2024-11-16T10:16:42,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/d479a914703741a3ad874288d127ff71 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/d479a914703741a3ad874288d127ff71 2024-11-16T10:16:42,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/8bed6301eac548ee85c8e4f4ac0437bc to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/8bed6301eac548ee85c8e4f4ac0437bc 2024-11-16T10:16:42,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/890fd37f70dc415292da867ee3388014 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/890fd37f70dc415292da867ee3388014 2024-11-16T10:16:42,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/b0b07aee41be40c0a608cb2031d3c8ee to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/b0b07aee41be40c0a608cb2031d3c8ee 2024-11-16T10:16:42,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/f0965fe515eb4a04a135d00bd2bdb796 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/f0965fe515eb4a04a135d00bd2bdb796 2024-11-16T10:16:42,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/08a402da80774697aadd0c8a589d671f to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/08a402da80774697aadd0c8a589d671f 2024-11-16T10:16:42,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/158a082610e04cf2878cd173e971d3c9 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/158a082610e04cf2878cd173e971d3c9 2024-11-16T10:16:42,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/88b52efad98847fdab171f03f59654af to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/88b52efad98847fdab171f03f59654af 2024-11-16T10:16:42,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/aab01e280c5e47a3acf69586e30a4f30 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/aab01e280c5e47a3acf69586e30a4f30 2024-11-16T10:16:42,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/e5927dcbf77b4c0c8ed21e48763b5be2 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/e5927dcbf77b4c0c8ed21e48763b5be2 2024-11-16T10:16:42,902 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/recovered.edits/376.seqid, newMaxSeqId=376, maxSeqId=1 2024-11-16T10:16:42,902 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. 2024-11-16T10:16:42,903 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 77e3e5908c8d586f7cad974ff9b02535: Waiting for close lock at 1731752201577Running coprocessor pre-close hooks at 1731752201577Disabling compacts and flushes for region at 1731752201577Disabling writes for close at 1731752201577Obtaining lock to block concurrent updates at 1731752201577Preparing flush snapshotting stores in 77e3e5908c8d586f7cad974ff9b02535 at 1731752201577Finished memstore snapshotting TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731752201578 (+1 ms)Flushing stores of TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535. at 1731752201578Flushing 77e3e5908c8d586f7cad974ff9b02535/A: creating writer at 1731752201578Flushing 77e3e5908c8d586f7cad974ff9b02535/A: appending metadata at 1731752201582 (+4 ms)Flushing 77e3e5908c8d586f7cad974ff9b02535/A: closing flushed file at 1731752201582Flushing 77e3e5908c8d586f7cad974ff9b02535/B: creating writer at 1731752201994 (+412 ms)Flushing 77e3e5908c8d586f7cad974ff9b02535/B: appending metadata at 1731752201997 (+3 ms)Flushing 77e3e5908c8d586f7cad974ff9b02535/B: closing flushed file at 1731752201997Flushing 77e3e5908c8d586f7cad974ff9b02535/C: creating writer at 1731752202407 (+410 ms)Flushing 77e3e5908c8d586f7cad974ff9b02535/C: appending metadata at 1731752202410 (+3 ms)Flushing 77e3e5908c8d586f7cad974ff9b02535/C: closing flushed file at 1731752202410Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39881cb7: reopening flushed file at 1731752202818 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@352a36aa: reopening flushed file at 1731752202822 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38acfeec: reopening flushed file at 1731752202826 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 77e3e5908c8d586f7cad974ff9b02535 in 1254ms, sequenceid=373, compaction requested=true at 1731752202831 (+5 ms)Writing region close event to WAL at 1731752202899 (+68 ms)Running coprocessor post-close hooks at 1731752202902 (+3 ms)Closed at 1731752202902 2024-11-16T10:16:42,905 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:42,905 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=77e3e5908c8d586f7cad974ff9b02535, regionState=CLOSED 2024-11-16T10:16:42,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 because future has completed 2024-11-16T10:16:42,910 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-16T10:16:42,910 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 77e3e5908c8d586f7cad974ff9b02535, server=b16ff38458c1,45417,1731752048912 in 1.4840 sec 2024-11-16T10:16:42,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-16T10:16:42,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77e3e5908c8d586f7cad974ff9b02535, UNASSIGN in 1.4900 sec 2024-11-16T10:16:42,915 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-16T10:16:42,915 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4960 sec 2024-11-16T10:16:42,917 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752202916"}]},"ts":"1731752202916"} 2024-11-16T10:16:42,918 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T10:16:42,919 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T10:16:42,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5110 sec 2024-11-16T10:16:43,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T10:16:43,565 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:43,566 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T10:16:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:16:43,571 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:16:43,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T10:16:43,573 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:16:43,577 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:43,580 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A, FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B, FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C, FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/recovered.edits] 2024-11-16T10:16:43,584 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/494c0b7d75a94d37b22327374c5c5b9e to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/494c0b7d75a94d37b22327374c5c5b9e 2024-11-16T10:16:43,585 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/baab6bfcde0c4feb9f644487a99d6eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/baab6bfcde0c4feb9f644487a99d6eab 2024-11-16T10:16:43,587 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/d68642d581a24fe182a8cd274c6adc13 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/A/d68642d581a24fe182a8cd274c6adc13 2024-11-16T10:16:43,589 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/7a5d85b3769e4193aaac32096bb41377 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/7a5d85b3769e4193aaac32096bb41377 2024-11-16T10:16:43,590 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/8764c382e7d94a468218558067e74843 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/8764c382e7d94a468218558067e74843 2024-11-16T10:16:43,591 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/8b3b4b50e60144948517b81445c8915d to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/B/8b3b4b50e60144948517b81445c8915d 2024-11-16T10:16:43,592 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/65c828e3cd2f461b9b9a6259142e72d6 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/65c828e3cd2f461b9b9a6259142e72d6 2024-11-16T10:16:43,593 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/6db90bada4ef48b29053ea18fdbd7a23 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/6db90bada4ef48b29053ea18fdbd7a23 2024-11-16T10:16:43,594 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/6f1aeb59da1145ccaf45e7726f8bfc9d to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/C/6f1aeb59da1145ccaf45e7726f8bfc9d 2024-11-16T10:16:43,596 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/recovered.edits/376.seqid to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535/recovered.edits/376.seqid 2024-11-16T10:16:43,596 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/77e3e5908c8d586f7cad974ff9b02535 2024-11-16T10:16:43,596 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T10:16:43,598 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:16:43,600 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T10:16:43,601 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T10:16:43,602 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:16:43,602 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T10:16:43,602 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731752203602"}]},"ts":"9223372036854775807"} 2024-11-16T10:16:43,604 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T10:16:43,604 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 77e3e5908c8d586f7cad974ff9b02535, NAME => 'TestAcidGuarantees,,1731752176257.77e3e5908c8d586f7cad974ff9b02535.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T10:16:43,604 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T10:16:43,604 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731752203604"}]},"ts":"9223372036854775807"} 2024-11-16T10:16:43,605 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T10:16:43,606 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:16:43,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 39 msec 2024-11-16T10:16:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T10:16:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T10:16:43,685 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:43,695 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 248) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=369 (was 402), ProcessCount=11 (was 11), AvailableMemoryMB=3698 (was 3690) - AvailableMemoryMB LEAK? - 2024-11-16T10:16:43,706 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=369, ProcessCount=11, AvailableMemoryMB=3698 2024-11-16T10:16:43,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T10:16:43,707 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T10:16:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T10:16:43,709 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T10:16:43,709 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:43,709 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-11-16T10:16:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-16T10:16:43,710 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T10:16:43,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742358_1534 (size=1061) 2024-11-16T10:16:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-16T10:16:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-16T10:16:44,121 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0 2024-11-16T10:16:44,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742359_1535 (size=53) 2024-11-16T10:16:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-16T10:16:44,533 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:16:44,533 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 571a63535798b9dbe219ba8ef9231eab, disabling compactions & flushes 2024-11-16T10:16:44,533 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:44,533 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:44,533 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. after waiting 0 ms 2024-11-16T10:16:44,534 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:44,534 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:44,534 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 571a63535798b9dbe219ba8ef9231eab: Waiting for close lock at 1731752204533Disabling compacts and flushes for region at 1731752204533Disabling writes for close at 1731752204533Writing region close event to WAL at 1731752204534 (+1 ms)Closed at 1731752204534 2024-11-16T10:16:44,537 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T10:16:44,538 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731752204537"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731752204537"}]},"ts":"1731752204537"} 2024-11-16T10:16:44,542 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T10:16:44,543 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T10:16:44,543 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752204543"}]},"ts":"1731752204543"} 2024-11-16T10:16:44,545 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T10:16:44,545 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=571a63535798b9dbe219ba8ef9231eab, ASSIGN}] 2024-11-16T10:16:44,547 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=571a63535798b9dbe219ba8ef9231eab, ASSIGN 2024-11-16T10:16:44,548 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=571a63535798b9dbe219ba8ef9231eab, ASSIGN; state=OFFLINE, location=b16ff38458c1,45417,1731752048912; forceNewPlan=false, retain=false 2024-11-16T10:16:44,698 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=571a63535798b9dbe219ba8ef9231eab, regionState=OPENING, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:44,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=571a63535798b9dbe219ba8ef9231eab, ASSIGN because future has completed 2024-11-16T10:16:44,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912}] 2024-11-16T10:16:44,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-16T10:16:44,858 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:44,859 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:16:44,859 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:44,859 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:16:44,860 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:44,860 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:44,862 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:44,865 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:16:44,865 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 571a63535798b9dbe219ba8ef9231eab columnFamilyName A 2024-11-16T10:16:44,865 DEBUG [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:44,866 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] regionserver.HStore(327): Store=571a63535798b9dbe219ba8ef9231eab/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:16:44,867 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:44,869 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:16:44,869 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 571a63535798b9dbe219ba8ef9231eab columnFamilyName B 2024-11-16T10:16:44,869 DEBUG [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:44,870 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] regionserver.HStore(327): Store=571a63535798b9dbe219ba8ef9231eab/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:16:44,871 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:44,873 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:16:44,873 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 571a63535798b9dbe219ba8ef9231eab columnFamilyName C 2024-11-16T10:16:44,873 DEBUG [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:44,874 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] regionserver.HStore(327): Store=571a63535798b9dbe219ba8ef9231eab/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:16:44,875 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:44,875 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:44,876 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:44,876 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:44,877 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:44,877 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:44,877 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:16:44,878 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:44,880 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T10:16:44,880 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened 571a63535798b9dbe219ba8ef9231eab; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67740190, jitterRate=0.009407490491867065}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:16:44,880 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:44,881 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for 571a63535798b9dbe219ba8ef9231eab: Running coprocessor pre-open hook at 1731752204860Writing region info on filesystem at 1731752204860Initializing all the Stores at 1731752204861 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752204862 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752204862Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752204862Cleaning up temporary data from old regions at 1731752204877 (+15 ms)Running coprocessor post-open hooks at 1731752204880 (+3 ms)Region opened successfully at 1731752204881 (+1 ms) 2024-11-16T10:16:44,882 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., pid=144, masterSystemTime=1731752204851 2024-11-16T10:16:44,883 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:44,883 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:44,884 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=571a63535798b9dbe219ba8ef9231eab, regionState=OPEN, openSeqNum=2, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:44,885 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 because future has completed 2024-11-16T10:16:44,887 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-16T10:16:44,887 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 in 186 msec 2024-11-16T10:16:44,889 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-16T10:16:44,889 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=571a63535798b9dbe219ba8ef9231eab, ASSIGN in 342 msec 2024-11-16T10:16:44,890 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T10:16:44,890 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752204890"}]},"ts":"1731752204890"} 2024-11-16T10:16:44,892 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T10:16:44,893 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T10:16:44,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1860 sec 2024-11-16T10:16:45,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-16T10:16:45,854 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:45,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d463f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:45,855 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:16:45,855 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:16:45,856 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:16:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:16:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:16:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c812665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:16:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:16:45,857 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:45,857 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:16:45,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1410b70e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:45,858 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:16:45,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:16:45,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:16:45,860 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:16:45,860 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T10:16:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is b16ff38458c1,45949,1731752047888 2024-11-16T10:16:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6b1a96bf 2024-11-16T10:16:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T10:16:45,862 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T10:16:45,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T10:16:45,864 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T10:16:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T10:16:45,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742360_1536 (size=1097) 2024-11-16T10:16:46,275 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-16T10:16:46,275 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-16T10:16:46,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T10:16:46,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=571a63535798b9dbe219ba8ef9231eab, REOPEN/MOVE}] 2024-11-16T10:16:46,279 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=571a63535798b9dbe219ba8ef9231eab, REOPEN/MOVE 2024-11-16T10:16:46,280 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=571a63535798b9dbe219ba8ef9231eab, regionState=CLOSING, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:46,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=571a63535798b9dbe219ba8ef9231eab, REOPEN/MOVE because future has completed 2024-11-16T10:16:46,282 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T10:16:46,282 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912}] 2024-11-16T10:16:46,434 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:46,434 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T10:16:46,434 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing 571a63535798b9dbe219ba8ef9231eab, disabling compactions & flushes 2024-11-16T10:16:46,434 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:46,434 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:46,434 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. after waiting 0 ms 2024-11-16T10:16:46,434 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:46,438 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T10:16:46,438 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:46,438 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for 571a63535798b9dbe219ba8ef9231eab: Waiting for close lock at 1731752206434Running coprocessor pre-close hooks at 1731752206434Disabling compacts and flushes for region at 1731752206434Disabling writes for close at 1731752206434Writing region close event to WAL at 1731752206435 (+1 ms)Running coprocessor post-close hooks at 1731752206438 (+3 ms)Closed at 1731752206438 2024-11-16T10:16:46,438 WARN [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: 571a63535798b9dbe219ba8ef9231eab to self. 2024-11-16T10:16:46,440 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:46,441 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=571a63535798b9dbe219ba8ef9231eab, regionState=CLOSED 2024-11-16T10:16:46,443 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 because future has completed 2024-11-16T10:16:46,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-16T10:16:46,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure 571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 in 161 msec 2024-11-16T10:16:46,447 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=571a63535798b9dbe219ba8ef9231eab, REOPEN/MOVE; state=CLOSED, location=b16ff38458c1,45417,1731752048912; forceNewPlan=false, retain=true 2024-11-16T10:16:46,597 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=571a63535798b9dbe219ba8ef9231eab, regionState=OPENING, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:46,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=571a63535798b9dbe219ba8ef9231eab, REOPEN/MOVE because future has completed 2024-11-16T10:16:46,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912}] 2024-11-16T10:16:46,753 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:46,753 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:16:46,753 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:46,753 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:16:46,753 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:46,753 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:46,754 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:46,755 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:16:46,755 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 571a63535798b9dbe219ba8ef9231eab columnFamilyName A 2024-11-16T10:16:46,756 DEBUG [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:46,756 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] regionserver.HStore(327): Store=571a63535798b9dbe219ba8ef9231eab/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:16:46,757 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:46,757 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:16:46,758 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 571a63535798b9dbe219ba8ef9231eab columnFamilyName B 2024-11-16T10:16:46,758 DEBUG [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:46,758 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] regionserver.HStore(327): Store=571a63535798b9dbe219ba8ef9231eab/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:16:46,758 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:46,759 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T10:16:46,759 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 571a63535798b9dbe219ba8ef9231eab columnFamilyName C 2024-11-16T10:16:46,759 DEBUG [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:46,759 INFO [StoreOpener-571a63535798b9dbe219ba8ef9231eab-1 {}] regionserver.HStore(327): Store=571a63535798b9dbe219ba8ef9231eab/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:16:46,759 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:46,759 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:46,760 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:46,761 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:46,762 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:46,762 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:46,762 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:16:46,763 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:46,764 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened 571a63535798b9dbe219ba8ef9231eab; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63671523, jitterRate=-0.0512203723192215}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:16:46,764 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:46,765 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for 571a63535798b9dbe219ba8ef9231eab: Running coprocessor pre-open hook at 1731752206753Writing region info on filesystem at 1731752206753Initializing all the Stores at 1731752206754 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752206754Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752206754Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752206754Cleaning up temporary data from old regions at 1731752206762 (+8 ms)Running coprocessor post-open hooks at 1731752206764 (+2 ms)Region opened successfully at 1731752206765 (+1 ms) 2024-11-16T10:16:46,766 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., pid=149, masterSystemTime=1731752206750 2024-11-16T10:16:46,767 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:46,768 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:46,768 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=571a63535798b9dbe219ba8ef9231eab, regionState=OPEN, openSeqNum=5, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:46,770 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 because future has completed 2024-11-16T10:16:46,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-11-16T10:16:46,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 in 172 msec 2024-11-16T10:16:46,774 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-16T10:16:46,774 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=571a63535798b9dbe219ba8ef9231eab, REOPEN/MOVE in 493 msec 2024-11-16T10:16:46,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-16T10:16:46,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 498 msec 2024-11-16T10:16:46,779 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 914 msec 2024-11-16T10:16:46,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-16T10:16:46,780 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:46,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26378a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,781 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:16:46,781 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:16:46,782 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:16:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:16:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:16:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57fecc11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:16:46,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:16:46,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:46,784 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:16:46,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d1a3c96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,785 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:16:46,785 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:16:46,785 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:16:46,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:16:46,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:16:46,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d2ce199, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:16:46,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:16:46,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:46,787 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:16:46,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@baf0838, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,789 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:16:46,789 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:16:46,790 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:16:46,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:16:46,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:16:46,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7938f0f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:16:46,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:16:46,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:46,791 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:16:46,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1acab269, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,792 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:16:46,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:16:46,793 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:16:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:16:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:16:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7737f1bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:16:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:16:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:46,795 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:16:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc02327, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,796 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:16:46,796 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:16:46,797 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:16:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:16:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:16:46,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@612ffa72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:16:46,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:16:46,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:46,799 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:16:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176e57fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,804 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:16:46,804 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:16:46,805 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:16:46,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:16:46,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:16:46,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349b714d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:16:46,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:16:46,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:46,807 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:16:46,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a85087d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,808 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:16:46,809 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:16:46,809 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:16:46,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:16:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:16:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b5df76c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:16:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:16:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:46,811 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:16:46,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f6ee24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,820 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:16:46,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:16:46,821 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:16:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:16:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:16:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e5c3e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:16:46,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:16:46,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:46,823 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:16:46,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@707279ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,824 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:16:46,824 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:16:46,825 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:16:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:16:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:16:46,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23fb412d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:16:46,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:16:46,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:46,827 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:16:46,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31157b95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e766e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,832 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,45949,-1 for getting cluster id 2024-11-16T10:16:46,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:16:46,834 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4ee6ab6-3ad8-4552-8e23-aafe5d412c59' 2024-11-16T10:16:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:16:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4ee6ab6-3ad8-4552-8e23-aafe5d412c59" 2024-11-16T10:16:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25ef2dcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,45949,-1] 2024-11-16T10:16:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:16:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:16:46,835 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:16:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17378d81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:16:46,840 DEBUG [Thread-2403 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:16:46,840 DEBUG [Thread-2398 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:16:46,841 DEBUG [Thread-2390 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:16:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:16:46,841 DEBUG [Thread-2398 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:16:46,842 DEBUG [Thread-2398 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:16:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:16:46,842 DEBUG [Thread-2390 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:16:46,842 DEBUG [Thread-2390 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:16:46,843 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:16:46,843 DEBUG [Thread-2405 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:16:46,843 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:16:46,843 DEBUG [Thread-2396 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:16:46,844 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:16:46,844 DEBUG [Thread-2396 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:16:46,844 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:16:46,844 DEBUG [Thread-2396 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:16:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-16T10:16:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:16:46,846 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:16:46,846 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:16:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:16:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:16:46,847 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:16:46,847 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:16:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T10:16:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:16:46,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:16:46,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:16:46,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:16:46,853 DEBUG [Thread-2392 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:16:46,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:16:46,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:16:46,855 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:16:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:16:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:16:46,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:46,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 571a63535798b9dbe219ba8ef9231eab 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T10:16:46,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=A 2024-11-16T10:16:46,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:46,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=B 2024-11-16T10:16:46,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:46,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=C 2024-11-16T10:16:46,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:46,860 DEBUG [Thread-2409 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:16:46,863 DEBUG [Thread-2394 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:16:46,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:16:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:16:46,865 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:16:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:16:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:16:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:16:46,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:16:46,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:16:46,878 DEBUG [Thread-2401 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:16:46,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:16:46,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:16:46,881 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:16:46,882 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:16:46,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:46,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43504 deadline: 1731752266881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:46,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:46,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:46,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:46,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752206874, max timestamp=1731752206884Num uniques 859; 2024-11-16T10:16:46,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752206874, max timestamp=1731752206884Num uniques 859; 2024-11-16T10:16:46,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:16:46,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:16:46,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752206874, max timestamp=1731752206883Num uniques 859; 2024-11-16T10:16:46,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:16:46,908 DEBUG [Thread-2407 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:16:46,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,45417,1731752048912, seqNum=-1] 2024-11-16T10:16:46,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:16:46,910 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:16:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5] 2024-11-16T10:16:46,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:46,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752266919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:46,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:46,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752266920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:46,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167257fba6f319458d8186a002b51f86cb_571a63535798b9dbe219ba8ef9231eab is 50, key is test_row_0/A:col10/1731752206848/Put/seqid=0 2024-11-16T10:16:46,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:46,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:46,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:46,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:46,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752266924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:46,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752266926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:46,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T10:16:46,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742361_1537 (size=12154) 2024-11-16T10:16:46,987 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:46,990 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167257fba6f319458d8186a002b51f86cb_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167257fba6f319458d8186a002b51f86cb_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:46,991 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/8c9d3d7d22ff40d0873213dd67b8bfd2, store: [table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:46,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/8c9d3d7d22ff40d0873213dd67b8bfd2 is 175, key is test_row_0/A:col10/1731752206848/Put/seqid=0 2024-11-16T10:16:47,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:16:47,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:47,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:47,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:47,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43504 deadline: 1731752266994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:47,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:47,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:47,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:47,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:47,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:47,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742362_1538 (size=30955) 2024-11-16T10:16:47,031 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/8c9d3d7d22ff40d0873213dd67b8bfd2 2024-11-16T10:16:47,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:47,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752267035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:47,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:47,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:47,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752267044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:47,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752267044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:47,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:47,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752267044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:47,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/d9b5a98409fb4e8b88a431f32e8384be is 50, key is test_row_0/B:col10/1731752206848/Put/seqid=0 2024-11-16T10:16:47,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742363_1539 (size=12001) 2024-11-16T10:16:47,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/d9b5a98409fb4e8b88a431f32e8384be 2024-11-16T10:16:47,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/51e2a975fc964a8885cf83af67693b4b is 50, key is test_row_0/C:col10/1731752206848/Put/seqid=0 2024-11-16T10:16:47,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:16:47,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:47,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:47,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:47,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:47,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:47,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:47,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T10:16:47,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742364_1540 (size=12001) 2024-11-16T10:16:47,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/51e2a975fc964a8885cf83af67693b4b 2024-11-16T10:16:47,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/8c9d3d7d22ff40d0873213dd67b8bfd2 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/8c9d3d7d22ff40d0873213dd67b8bfd2 2024-11-16T10:16:47,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/8c9d3d7d22ff40d0873213dd67b8bfd2, entries=150, sequenceid=15, filesize=30.2 K 2024-11-16T10:16:47,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/d9b5a98409fb4e8b88a431f32e8384be as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/d9b5a98409fb4e8b88a431f32e8384be 2024-11-16T10:16:47,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/d9b5a98409fb4e8b88a431f32e8384be, entries=150, sequenceid=15, filesize=11.7 K 2024-11-16T10:16:47,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/51e2a975fc964a8885cf83af67693b4b as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/51e2a975fc964a8885cf83af67693b4b 2024-11-16T10:16:47,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:47,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43504 deadline: 1731752267205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:47,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:47,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/51e2a975fc964a8885cf83af67693b4b, entries=150, sequenceid=15, filesize=11.7 K 2024-11-16T10:16:47,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 571a63535798b9dbe219ba8ef9231eab in 356ms, sequenceid=15, compaction requested=false 2024-11-16T10:16:47,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:47,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:47,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 571a63535798b9dbe219ba8ef9231eab 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-16T10:16:47,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=A 2024-11-16T10:16:47,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:47,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=B 2024-11-16T10:16:47,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:47,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=C 2024-11-16T10:16:47,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:47,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169a9a8a5dc51f47a8b1957aade4cb80bb_571a63535798b9dbe219ba8ef9231eab is 50, key is test_row_0/A:col10/1731752206916/Put/seqid=0 2024-11-16T10:16:47,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:47,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752267262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:47,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:47,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:47,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752267267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:47,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:47,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752267269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:47,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752267270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:47,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742365_1541 (size=14594) 2024-11-16T10:16:47,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:16:47,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:47,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:47,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:47,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:47,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:47,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:47,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752267374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:47,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:47,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752267384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752267384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:47,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752267384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:47,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:16:47,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:47,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:47,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:47,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:47,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:47,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T10:16:47,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43504 deadline: 1731752267514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:47,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752267586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:47,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:47,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752267594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:47,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:47,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752267594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:47,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:47,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752267595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:47,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:47,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:47,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:16:47,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:47,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:47,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:47,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:47,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:47,689 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:47,693 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169a9a8a5dc51f47a8b1957aade4cb80bb_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169a9a8a5dc51f47a8b1957aade4cb80bb_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:47,694 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/baee30d5060b41599c28798bfb448211, store: [table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:47,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/baee30d5060b41599c28798bfb448211 is 175, key is test_row_0/A:col10/1731752206916/Put/seqid=0 2024-11-16T10:16:47,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742366_1542 (size=39549) 2024-11-16T10:16:47,705 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/baee30d5060b41599c28798bfb448211 2024-11-16T10:16:47,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/54d456119f954dc6961f74b8ff6f9427 is 50, key is test_row_0/B:col10/1731752206916/Put/seqid=0 2024-11-16T10:16:47,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742367_1543 (size=12001) 2024-11-16T10:16:47,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:16:47,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:47,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:47,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:47,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:47,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:47,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:47,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752267894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:47,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:47,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:47,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:47,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752267905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:47,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752267905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:47,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752267905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:47,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:16:47,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:47,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:47,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:47,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:47,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T10:16:48,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:48,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43504 deadline: 1731752268024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:48,026 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,026 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,026 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:48,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:16:48,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:48,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:48,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:48,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:48,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:48,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:48,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/54d456119f954dc6961f74b8ff6f9427 2024-11-16T10:16:48,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/b6bbc15cad8c48fabb5a6f1afa62f619 is 50, key is test_row_0/C:col10/1731752206916/Put/seqid=0 2024-11-16T10:16:48,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742368_1544 (size=12001) 2024-11-16T10:16:48,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/b6bbc15cad8c48fabb5a6f1afa62f619 2024-11-16T10:16:48,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/baee30d5060b41599c28798bfb448211 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/baee30d5060b41599c28798bfb448211 2024-11-16T10:16:48,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/baee30d5060b41599c28798bfb448211, entries=200, sequenceid=52, filesize=38.6 K 2024-11-16T10:16:48,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/54d456119f954dc6961f74b8ff6f9427 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/54d456119f954dc6961f74b8ff6f9427 2024-11-16T10:16:48,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/54d456119f954dc6961f74b8ff6f9427, entries=150, sequenceid=52, filesize=11.7 K 2024-11-16T10:16:48,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/b6bbc15cad8c48fabb5a6f1afa62f619 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/b6bbc15cad8c48fabb5a6f1afa62f619 2024-11-16T10:16:48,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/b6bbc15cad8c48fabb5a6f1afa62f619, entries=150, sequenceid=52, filesize=11.7 K 2024-11-16T10:16:48,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for 571a63535798b9dbe219ba8ef9231eab in 924ms, sequenceid=52, compaction requested=false 2024-11-16T10:16:48,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:48,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:16:48,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:48,232 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 571a63535798b9dbe219ba8ef9231eab 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T10:16:48,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=A 2024-11-16T10:16:48,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:48,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=B 2024-11-16T10:16:48,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:48,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=C 2024-11-16T10:16:48,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:48,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111674c48793783c4f0d8d90cef9942bf767_571a63535798b9dbe219ba8ef9231eab is 50, key is test_row_0/A:col10/1731752207267/Put/seqid=0 2024-11-16T10:16:48,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742369_1545 (size=12154) 2024-11-16T10:16:48,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:48,250 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111674c48793783c4f0d8d90cef9942bf767_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111674c48793783c4f0d8d90cef9942bf767_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:48,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/e0d2b236ae304176bc3f6928d18af6af, store: [table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:48,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/e0d2b236ae304176bc3f6928d18af6af is 175, key is test_row_0/A:col10/1731752207267/Put/seqid=0 2024-11-16T10:16:48,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742370_1546 (size=30955) 2024-11-16T10:16:48,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:48,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:48,433 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T10:16:48,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752208431, max timestamp=1731752208440Num uniques 859; 2024-11-16T10:16:48,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752208431, max timestamp=1731752208440Num uniques 859; 2024-11-16T10:16:48,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:16:48,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:16:48,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752208431, max timestamp=1731752208440Num uniques 859; 2024-11-16T10:16:48,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:16:48,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752268465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:48,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752268467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:48,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:48,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:48,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752268471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:48,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:48,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752268472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:48,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:48,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:48,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752268573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:48,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:48,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752268584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:48,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752268584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:48,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752268584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:48,663 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/e0d2b236ae304176bc3f6928d18af6af 2024-11-16T10:16:48,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/323693d04fe54ba8a4e05ecf8f31d40d is 50, key is test_row_0/B:col10/1731752207267/Put/seqid=0 2024-11-16T10:16:48,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742371_1547 (size=12001) 2024-11-16T10:16:48,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752268784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:48,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:48,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:48,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752268793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:48,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:48,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:48,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752268793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:48,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:48,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:48,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752268794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:48,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:48,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T10:16:49,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:49,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43504 deadline: 1731752269044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:49,049 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_0' in 571a63535798b9dbe219ba8ef9231eab of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:16:49,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:49,073 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/323693d04fe54ba8a4e05ecf8f31d40d 2024-11-16T10:16:49,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/42f45e8b70f24585bf631e57cd30be95 is 50, key is test_row_0/C:col10/1731752207267/Put/seqid=0 2024-11-16T10:16:49,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742372_1548 (size=12001) 2024-11-16T10:16:49,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:49,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752269094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:49,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752269104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752269104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:49,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752269104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:49,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:49,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:49,496 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/42f45e8b70f24585bf631e57cd30be95 2024-11-16T10:16:49,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/e0d2b236ae304176bc3f6928d18af6af as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/e0d2b236ae304176bc3f6928d18af6af 2024-11-16T10:16:49,508 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/e0d2b236ae304176bc3f6928d18af6af, entries=150, sequenceid=63, filesize=30.2 K 2024-11-16T10:16:49,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/323693d04fe54ba8a4e05ecf8f31d40d as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/323693d04fe54ba8a4e05ecf8f31d40d 2024-11-16T10:16:49,530 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/323693d04fe54ba8a4e05ecf8f31d40d, entries=150, sequenceid=63, filesize=11.7 K 2024-11-16T10:16:49,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/42f45e8b70f24585bf631e57cd30be95 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/42f45e8b70f24585bf631e57cd30be95 2024-11-16T10:16:49,535 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/42f45e8b70f24585bf631e57cd30be95, entries=150, sequenceid=63, filesize=11.7 K 2024-11-16T10:16:49,539 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 571a63535798b9dbe219ba8ef9231eab in 1307ms, sequenceid=63, compaction requested=true 2024-11-16T10:16:49,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:49,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:49,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-16T10:16:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-16T10:16:49,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-16T10:16:49,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6940 sec 2024-11-16T10:16:49,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.7010 sec 2024-11-16T10:16:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:49,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 571a63535798b9dbe219ba8ef9231eab 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-16T10:16:49,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=A 2024-11-16T10:16:49,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T10:16:49,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=B 2024-11-16T10:16:49,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T10:16:49,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=C 2024-11-16T10:16:49,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T10:16:49,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116db6b539a0f1b41d58287031077d60e8e_571a63535798b9dbe219ba8ef9231eab is 50, key is test_row_0/A:col10/1731752209620/Put/seqid=0 2024-11-16T10:16:49,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742373_1549 (size=12154) 2024-11-16T10:16:49,651 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:49,655 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116db6b539a0f1b41d58287031077d60e8e_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116db6b539a0f1b41d58287031077d60e8e_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:49,656 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/3589724e517d4d818f5447d5e8fcf6b5, store: [table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:49,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/3589724e517d4d818f5447d5e8fcf6b5 is 175, key is test_row_0/A:col10/1731752209620/Put/seqid=0 2024-11-16T10:16:49,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:49,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752269649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752269652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:49,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:49,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752269652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752269652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:49,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742374_1550 (size=30955) 2024-11-16T10:16:49,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752269764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:49,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752269764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752269764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752269764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:49,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752269974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:49,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:49,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752269974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:49,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:49,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:49,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752269974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752269974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:50,066 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=102, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/3589724e517d4d818f5447d5e8fcf6b5 2024-11-16T10:16:50,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/1c5df67f322646ffa719cfdd5a0990e4 is 50, key is test_row_0/B:col10/1731752209620/Put/seqid=0 2024-11-16T10:16:50,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742375_1551 (size=12001) 2024-11-16T10:16:50,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/1c5df67f322646ffa719cfdd5a0990e4 2024-11-16T10:16:50,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/3d32caf57af441c3bc2bac37c990e2a5 is 50, key is test_row_0/C:col10/1731752209620/Put/seqid=0 2024-11-16T10:16:50,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742376_1552 (size=12001) 2024-11-16T10:16:50,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/3d32caf57af441c3bc2bac37c990e2a5 2024-11-16T10:16:50,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/3589724e517d4d818f5447d5e8fcf6b5 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/3589724e517d4d818f5447d5e8fcf6b5 2024-11-16T10:16:50,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/3589724e517d4d818f5447d5e8fcf6b5, entries=150, sequenceid=102, filesize=30.2 K 2024-11-16T10:16:50,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/1c5df67f322646ffa719cfdd5a0990e4 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/1c5df67f322646ffa719cfdd5a0990e4 2024-11-16T10:16:50,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/1c5df67f322646ffa719cfdd5a0990e4, entries=150, sequenceid=102, filesize=11.7 K 2024-11-16T10:16:50,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/3d32caf57af441c3bc2bac37c990e2a5 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3d32caf57af441c3bc2bac37c990e2a5 2024-11-16T10:16:50,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3d32caf57af441c3bc2bac37c990e2a5, entries=150, sequenceid=102, filesize=11.7 K 2024-11-16T10:16:50,123 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~425.48 KB/435696, currentSize=33.54 KB/34350 for 571a63535798b9dbe219ba8ef9231eab in 494ms, sequenceid=102, compaction requested=true 2024-11-16T10:16:50,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:50,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 571a63535798b9dbe219ba8ef9231eab:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:16:50,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:50,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 571a63535798b9dbe219ba8ef9231eab:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:16:50,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T10:16:50,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 571a63535798b9dbe219ba8ef9231eab:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:16:50,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-16T10:16:50,124 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:16:50,124 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:16:50,127 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:16:50,127 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:16:50,127 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 571a63535798b9dbe219ba8ef9231eab/C is initiating minor compaction (all files) 2024-11-16T10:16:50,127 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 571a63535798b9dbe219ba8ef9231eab/A is initiating minor compaction (all files) 2024-11-16T10:16:50,127 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 571a63535798b9dbe219ba8ef9231eab/C in TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:50,127 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 571a63535798b9dbe219ba8ef9231eab/A in TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:50,127 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/8c9d3d7d22ff40d0873213dd67b8bfd2, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/baee30d5060b41599c28798bfb448211, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/e0d2b236ae304176bc3f6928d18af6af, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/3589724e517d4d818f5447d5e8fcf6b5] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp, totalSize=129.3 K 2024-11-16T10:16:50,127 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/51e2a975fc964a8885cf83af67693b4b, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/b6bbc15cad8c48fabb5a6f1afa62f619, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/42f45e8b70f24585bf631e57cd30be95, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3d32caf57af441c3bc2bac37c990e2a5] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp, totalSize=46.9 K 2024-11-16T10:16:50,127 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:50,128 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. files: [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/8c9d3d7d22ff40d0873213dd67b8bfd2, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/baee30d5060b41599c28798bfb448211, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/e0d2b236ae304176bc3f6928d18af6af, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/3589724e517d4d818f5447d5e8fcf6b5] 2024-11-16T10:16:50,128 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c9d3d7d22ff40d0873213dd67b8bfd2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731752206848 2024-11-16T10:16:50,128 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 51e2a975fc964a8885cf83af67693b4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731752206848 2024-11-16T10:16:50,129 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting b6bbc15cad8c48fabb5a6f1afa62f619, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731752206916 2024-11-16T10:16:50,129 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting baee30d5060b41599c28798bfb448211, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731752206911 2024-11-16T10:16:50,129 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 42f45e8b70f24585bf631e57cd30be95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731752207261 2024-11-16T10:16:50,129 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0d2b236ae304176bc3f6928d18af6af, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731752207261 2024-11-16T10:16:50,129 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d32caf57af441c3bc2bac37c990e2a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731752208462 2024-11-16T10:16:50,129 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3589724e517d4d818f5447d5e8fcf6b5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731752208462 2024-11-16T10:16:50,136 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:50,137 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116d39f14241a0b4c5c915e19c576f9b428_571a63535798b9dbe219ba8ef9231eab store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:50,137 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 571a63535798b9dbe219ba8ef9231eab#C#compaction#471 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:50,138 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/61d24a7ba099460aa2da37464e22aa31 is 50, key is test_row_0/C:col10/1731752209620/Put/seqid=0 2024-11-16T10:16:50,140 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116d39f14241a0b4c5c915e19c576f9b428_571a63535798b9dbe219ba8ef9231eab, store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:50,140 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d39f14241a0b4c5c915e19c576f9b428_571a63535798b9dbe219ba8ef9231eab because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:50,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742377_1553 (size=12139) 2024-11-16T10:16:50,155 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/61d24a7ba099460aa2da37464e22aa31 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/61d24a7ba099460aa2da37464e22aa31 2024-11-16T10:16:50,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742378_1554 (size=4469) 2024-11-16T10:16:50,160 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 571a63535798b9dbe219ba8ef9231eab/C of 571a63535798b9dbe219ba8ef9231eab into 61d24a7ba099460aa2da37464e22aa31(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:50,160 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:50,160 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., storeName=571a63535798b9dbe219ba8ef9231eab/C, priority=12, startTime=1731752210124; duration=0sec 2024-11-16T10:16:50,160 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:50,160 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 571a63535798b9dbe219ba8ef9231eab:C 2024-11-16T10:16:50,160 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:16:50,162 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:16:50,162 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 571a63535798b9dbe219ba8ef9231eab/B is initiating minor compaction (all files) 2024-11-16T10:16:50,162 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 571a63535798b9dbe219ba8ef9231eab/B in TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:50,162 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/d9b5a98409fb4e8b88a431f32e8384be, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/54d456119f954dc6961f74b8ff6f9427, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/323693d04fe54ba8a4e05ecf8f31d40d, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/1c5df67f322646ffa719cfdd5a0990e4] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp, totalSize=46.9 K 2024-11-16T10:16:50,162 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting d9b5a98409fb4e8b88a431f32e8384be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731752206848 2024-11-16T10:16:50,163 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 54d456119f954dc6961f74b8ff6f9427, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731752206916 2024-11-16T10:16:50,163 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 323693d04fe54ba8a4e05ecf8f31d40d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731752207261 2024-11-16T10:16:50,163 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c5df67f322646ffa719cfdd5a0990e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731752208462 2024-11-16T10:16:50,174 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 571a63535798b9dbe219ba8ef9231eab#B#compaction#473 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:50,174 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/8a5e689a6a4042f996b57b4cf6328c37 is 50, key is test_row_0/B:col10/1731752209620/Put/seqid=0 2024-11-16T10:16:50,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742379_1555 (size=12139) 2024-11-16T10:16:50,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:50,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 571a63535798b9dbe219ba8ef9231eab 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T10:16:50,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=A 2024-11-16T10:16:50,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:50,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=B 2024-11-16T10:16:50,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:50,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=C 2024-11-16T10:16:50,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:50,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a0c5e08c21bd45deace084a9bf0294cf_571a63535798b9dbe219ba8ef9231eab is 50, key is test_row_0/A:col10/1731752210286/Put/seqid=0 2024-11-16T10:16:50,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742380_1556 (size=17034) 2024-11-16T10:16:50,298 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:50,304 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a0c5e08c21bd45deace084a9bf0294cf_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a0c5e08c21bd45deace084a9bf0294cf_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:50,305 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/317d9beb089e42c9a6f796d43a192249, store: [table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:50,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/317d9beb089e42c9a6f796d43a192249 is 175, key is test_row_0/A:col10/1731752210286/Put/seqid=0 2024-11-16T10:16:50,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742381_1557 (size=48139) 2024-11-16T10:16:50,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752210301, max timestamp=1731752210314Num uniques 859; 2024-11-16T10:16:50,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752210301, max timestamp=1731752210314Num uniques 859; 2024-11-16T10:16:50,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731752210301, max timestamp=1731752210314Num uniques 859; 2024-11-16T10:16:50,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:16:50,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:16:50,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T10:16:50,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:50,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752270334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:50,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:50,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752270335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:50,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752270337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:50,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752270337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:50,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:50,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752270454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:50,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:50,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:50,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752270454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:50,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:50,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:50,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:50,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752270454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:50,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752270454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:50,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:50,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:50,559 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 571a63535798b9dbe219ba8ef9231eab#A#compaction#472 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:50,560 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/1498a57065a1483bb5144cdf4d2db430 is 175, key is test_row_0/A:col10/1731752209620/Put/seqid=0 2024-11-16T10:16:50,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742382_1558 (size=31093) 2024-11-16T10:16:50,574 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/1498a57065a1483bb5144cdf4d2db430 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/1498a57065a1483bb5144cdf4d2db430 2024-11-16T10:16:50,585 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 571a63535798b9dbe219ba8ef9231eab/A of 571a63535798b9dbe219ba8ef9231eab into 1498a57065a1483bb5144cdf4d2db430(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:50,585 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:50,585 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., storeName=571a63535798b9dbe219ba8ef9231eab/A, priority=12, startTime=1731752210124; duration=0sec 2024-11-16T10:16:50,586 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:50,586 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 571a63535798b9dbe219ba8ef9231eab:A 2024-11-16T10:16:50,598 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/8a5e689a6a4042f996b57b4cf6328c37 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/8a5e689a6a4042f996b57b4cf6328c37 2024-11-16T10:16:50,603 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 571a63535798b9dbe219ba8ef9231eab/B of 571a63535798b9dbe219ba8ef9231eab into 8a5e689a6a4042f996b57b4cf6328c37(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:50,603 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:50,603 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., storeName=571a63535798b9dbe219ba8ef9231eab/B, priority=12, startTime=1731752210124; duration=0sec 2024-11-16T10:16:50,603 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:50,603 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 571a63535798b9dbe219ba8ef9231eab:B 2024-11-16T10:16:50,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:50,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752270664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:50,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:50,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:50,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752270664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:50,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:50,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752270664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:50,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:50,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752270664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:50,711 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=114, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/317d9beb089e42c9a6f796d43a192249 2024-11-16T10:16:50,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/8ce58593de324ebfa40a0ccd01c1204b is 50, key is test_row_0/B:col10/1731752210286/Put/seqid=0 2024-11-16T10:16:50,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742383_1559 (size=12001) 2024-11-16T10:16:50,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752270974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:50,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752270974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:50,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752270974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:50,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752270974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:51,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T10:16:51,004 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:51,005 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:16:51,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-16T10:16:51,008 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:16:51,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T10:16:51,010 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:16:51,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:16:51,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43504 deadline: 1731752271064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:51,069 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_0' in 571a63535798b9dbe219ba8ef9231eab of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:16:51,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:51,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:51,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T10:16:51,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/8ce58593de324ebfa40a0ccd01c1204b 2024-11-16T10:16:51,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/3b1396c3a2384b1f8068e4d2eeda7c92 is 50, key is test_row_0/C:col10/1731752210286/Put/seqid=0 2024-11-16T10:16:51,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742384_1560 (size=12001) 2024-11-16T10:16:51,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-16T10:16:51,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:51,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:51,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:51,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:51,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:51,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-16T10:16:51,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:51,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:51,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:51,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:51,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T10:16:51,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-16T10:16:51,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:51,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:51,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:51,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:51,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:51,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752271484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:51,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:51,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:51,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:51,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752271485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:51,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:51,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:51,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:51,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752271485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:51,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:51,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:51,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:51,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752271485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:51,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:51,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:51,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:51,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/3b1396c3a2384b1f8068e4d2eeda7c92 2024-11-16T10:16:51,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/317d9beb089e42c9a6f796d43a192249 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/317d9beb089e42c9a6f796d43a192249 2024-11-16T10:16:51,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/317d9beb089e42c9a6f796d43a192249, entries=250, sequenceid=114, filesize=47.0 K 2024-11-16T10:16:51,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/8ce58593de324ebfa40a0ccd01c1204b as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/8ce58593de324ebfa40a0ccd01c1204b 2024-11-16T10:16:51,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/8ce58593de324ebfa40a0ccd01c1204b, entries=150, sequenceid=114, filesize=11.7 K 2024-11-16T10:16:51,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/3b1396c3a2384b1f8068e4d2eeda7c92 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3b1396c3a2384b1f8068e4d2eeda7c92 2024-11-16T10:16:51,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3b1396c3a2384b1f8068e4d2eeda7c92, entries=150, sequenceid=114, filesize=11.7 K 2024-11-16T10:16:51,558 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 571a63535798b9dbe219ba8ef9231eab in 1271ms, sequenceid=114, compaction requested=false 2024-11-16T10:16:51,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:51,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-16T10:16:51,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:51,620 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 571a63535798b9dbe219ba8ef9231eab 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-16T10:16:51,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=A 2024-11-16T10:16:51,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:51,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=B 2024-11-16T10:16:51,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:51,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=C 2024-11-16T10:16:51,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T10:16:51,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164dc4fadf8be34915994acacee5e4a504_571a63535798b9dbe219ba8ef9231eab is 50, key is test_row_0/A:col10/1731752210335/Put/seqid=0 2024-11-16T10:16:51,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742385_1561 (size=12304) 2024-11-16T10:16:52,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,054 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164dc4fadf8be34915994acacee5e4a504_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164dc4fadf8be34915994acacee5e4a504_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:52,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/c4849065beee4411a9307bbb3d3a2be5, store: [table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:52,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/c4849065beee4411a9307bbb3d3a2be5 is 175, key is test_row_0/A:col10/1731752210335/Put/seqid=0 2024-11-16T10:16:52,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742386_1562 (size=31105) 2024-11-16T10:16:52,061 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/c4849065beee4411a9307bbb3d3a2be5 2024-11-16T10:16:52,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/9ca8869e43e24c0e80d6cb18a218d467 is 50, key is test_row_0/B:col10/1731752210335/Put/seqid=0 2024-11-16T10:16:52,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742387_1563 (size=12151) 2024-11-16T10:16:52,094 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/9ca8869e43e24c0e80d6cb18a218d467 2024-11-16T10:16:52,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/53aa52d5d5574d0ca1f3d34d734740fb is 50, key is test_row_0/C:col10/1731752210335/Put/seqid=0 2024-11-16T10:16:52,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742388_1564 (size=12151) 2024-11-16T10:16:52,118 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/53aa52d5d5574d0ca1f3d34d734740fb 2024-11-16T10:16:52,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/c4849065beee4411a9307bbb3d3a2be5 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/c4849065beee4411a9307bbb3d3a2be5 2024-11-16T10:16:52,130 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/c4849065beee4411a9307bbb3d3a2be5, entries=150, sequenceid=151, filesize=30.4 K 2024-11-16T10:16:52,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/9ca8869e43e24c0e80d6cb18a218d467 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/9ca8869e43e24c0e80d6cb18a218d467 2024-11-16T10:16:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,136 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/9ca8869e43e24c0e80d6cb18a218d467, entries=150, sequenceid=151, filesize=11.9 K 2024-11-16T10:16:52,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T10:16:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/53aa52d5d5574d0ca1f3d34d734740fb as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/53aa52d5d5574d0ca1f3d34d734740fb 2024-11-16T10:16:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,152 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/53aa52d5d5574d0ca1f3d34d734740fb, entries=150, sequenceid=151, filesize=11.9 K 2024-11-16T10:16:52,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,153 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=0 B/0 for 571a63535798b9dbe219ba8ef9231eab in 533ms, sequenceid=151, compaction requested=true 2024-11-16T10:16:52,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:52,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:52,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-16T10:16:52,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-16T10:16:52,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-16T10:16:52,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1440 sec 2024-11-16T10:16:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.1520 sec 2024-11-16T10:16:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 571a63535798b9dbe219ba8ef9231eab 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T10:16:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=A 2024-11-16T10:16:52,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:52,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=B 2024-11-16T10:16:52,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:52,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=C 2024-11-16T10:16:52,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116841e98ff7dc04995b8daa734c1f44f92_571a63535798b9dbe219ba8ef9231eab is 50, key is test_row_0/A:col10/1731752212559/Put/seqid=0 2024-11-16T10:16:52,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742389_1565 (size=17284) 2024-11-16T10:16:52,604 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:52,607 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116841e98ff7dc04995b8daa734c1f44f92_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116841e98ff7dc04995b8daa734c1f44f92_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:52,608 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/74f50b8912a24a0598801681ad0d1a6b, store: [table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:52,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/74f50b8912a24a0598801681ad0d1a6b is 175, key is test_row_0/A:col10/1731752212559/Put/seqid=0 2024-11-16T10:16:52,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742390_1566 (size=48389) 2024-11-16T10:16:52,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752272645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:52,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752212608, max timestamp=1731752212648Num uniques 859; 2024-11-16T10:16:52,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752212608, max timestamp=1731752212648Num uniques 859; 2024-11-16T10:16:52,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752212608, max timestamp=1731752212648Num uniques 859; 2024-11-16T10:16:52,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:16:52,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:16:52,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:16:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:52,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752272690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:52,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:52,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:52,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:52,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752272691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:52,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752272692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:52,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:52,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:52,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:52,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:52,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:52,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:52,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752272754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:52,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:52,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:52,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:52,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752272805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:52,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752272805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:52,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752272805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:52,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:52,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:52,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:52,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752272964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:52,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:52,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:52,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:53,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:53,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752273014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:53,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:53,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752273014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:53,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:53,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:53,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752273014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:53,019 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/74f50b8912a24a0598801681ad0d1a6b 2024-11-16T10:16:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:53,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/7f2748e818434a8e81307bc972a8230f is 50, key is test_row_0/B:col10/1731752212559/Put/seqid=0 2024-11-16T10:16:53,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742391_1567 (size=12151) 2024-11-16T10:16:53,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/7f2748e818434a8e81307bc972a8230f 2024-11-16T10:16:53,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/3a543f85da214aceb97e03f21f82b0a3 is 50, key is test_row_0/C:col10/1731752212559/Put/seqid=0 2024-11-16T10:16:53,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742392_1568 (size=12151) 2024-11-16T10:16:53,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T10:16:53,154 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:53,155 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:16:53,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-16T10:16:53,156 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:16:53,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T10:16:53,157 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:16:53,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:16:53,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T10:16:53,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:53,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752273274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:53,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T10:16:53,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:53,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:53,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:53,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:53,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:53,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752273324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752273324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752273324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:53,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T10:16:53,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:53,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:53,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:53,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:53,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:53,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:53,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T10:16:53,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/3a543f85da214aceb97e03f21f82b0a3 2024-11-16T10:16:53,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/74f50b8912a24a0598801681ad0d1a6b as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/74f50b8912a24a0598801681ad0d1a6b 2024-11-16T10:16:53,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/74f50b8912a24a0598801681ad0d1a6b, entries=250, sequenceid=161, filesize=47.3 K 2024-11-16T10:16:53,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/7f2748e818434a8e81307bc972a8230f as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/7f2748e818434a8e81307bc972a8230f 2024-11-16T10:16:53,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/7f2748e818434a8e81307bc972a8230f, entries=150, sequenceid=161, filesize=11.9 K 2024-11-16T10:16:53,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/3a543f85da214aceb97e03f21f82b0a3 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3a543f85da214aceb97e03f21f82b0a3 2024-11-16T10:16:53,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3a543f85da214aceb97e03f21f82b0a3, entries=150, sequenceid=161, filesize=11.9 K 2024-11-16T10:16:53,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 571a63535798b9dbe219ba8ef9231eab in 973ms, sequenceid=161, compaction requested=true 2024-11-16T10:16:53,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:53,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 571a63535798b9dbe219ba8ef9231eab:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:16:53,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:53,534 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:16:53,534 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:16:53,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 571a63535798b9dbe219ba8ef9231eab:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:16:53,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:53,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 571a63535798b9dbe219ba8ef9231eab:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:16:53,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:53,535 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:16:53,535 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:16:53,535 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 571a63535798b9dbe219ba8ef9231eab/B is initiating minor compaction (all files) 2024-11-16T10:16:53,535 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 571a63535798b9dbe219ba8ef9231eab/A is initiating minor compaction (all files) 2024-11-16T10:16:53,535 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 571a63535798b9dbe219ba8ef9231eab/B in TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:53,535 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/8a5e689a6a4042f996b57b4cf6328c37, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/8ce58593de324ebfa40a0ccd01c1204b, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/9ca8869e43e24c0e80d6cb18a218d467, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/7f2748e818434a8e81307bc972a8230f] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp, totalSize=47.3 K 2024-11-16T10:16:53,535 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 571a63535798b9dbe219ba8ef9231eab/A in TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:53,536 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/1498a57065a1483bb5144cdf4d2db430, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/317d9beb089e42c9a6f796d43a192249, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/c4849065beee4411a9307bbb3d3a2be5, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/74f50b8912a24a0598801681ad0d1a6b] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp, totalSize=155.0 K 2024-11-16T10:16:53,536 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:53,536 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. files: [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/1498a57065a1483bb5144cdf4d2db430, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/317d9beb089e42c9a6f796d43a192249, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/c4849065beee4411a9307bbb3d3a2be5, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/74f50b8912a24a0598801681ad0d1a6b] 2024-11-16T10:16:53,536 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a5e689a6a4042f996b57b4cf6328c37, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731752208462 2024-11-16T10:16:53,536 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1498a57065a1483bb5144cdf4d2db430, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731752208462 2024-11-16T10:16:53,536 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ce58593de324ebfa40a0ccd01c1204b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731752209651 2024-11-16T10:16:53,537 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ca8869e43e24c0e80d6cb18a218d467, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731752210333 2024-11-16T10:16:53,537 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 317d9beb089e42c9a6f796d43a192249, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731752209648 2024-11-16T10:16:53,537 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f2748e818434a8e81307bc972a8230f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731752212539 2024-11-16T10:16:53,537 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4849065beee4411a9307bbb3d3a2be5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731752210333 2024-11-16T10:16:53,537 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74f50b8912a24a0598801681ad0d1a6b, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731752212530 2024-11-16T10:16:53,551 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 571a63535798b9dbe219ba8ef9231eab#B#compaction#483 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:53,551 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/fe509def5cc74795a7e08e44058440ba is 50, key is test_row_0/B:col10/1731752212559/Put/seqid=0 2024-11-16T10:16:53,571 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:53,587 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111650a2f9c057c1471095bb003a5bfbbe53_571a63535798b9dbe219ba8ef9231eab store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:53,590 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111650a2f9c057c1471095bb003a5bfbbe53_571a63535798b9dbe219ba8ef9231eab, store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:53,590 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111650a2f9c057c1471095bb003a5bfbbe53_571a63535798b9dbe219ba8ef9231eab because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:53,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742393_1569 (size=12425) 2024-11-16T10:16:53,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742394_1570 (size=4469) 2024-11-16T10:16:53,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T10:16:53,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:53,615 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 571a63535798b9dbe219ba8ef9231eab 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-16T10:16:53,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=A 2024-11-16T10:16:53,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:53,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=B 2024-11-16T10:16:53,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:53,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=C 2024-11-16T10:16:53,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:53,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161d47619cd3784c47a34be52300942b43_571a63535798b9dbe219ba8ef9231eab is 50, key is test_row_0/A:col10/1731752212691/Put/seqid=0 2024-11-16T10:16:53,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742395_1571 (size=12304) 2024-11-16T10:16:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T10:16:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:53,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:53,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752273834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:53,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752273834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:53,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:53,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:53,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752273837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:53,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:53,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752273840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:53,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752273954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:53,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752273954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:53,994 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 571a63535798b9dbe219ba8ef9231eab#A#compaction#484 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:53,995 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/cca0b2726f8445008e7c205bdcb9862f is 175, key is test_row_0/A:col10/1731752212559/Put/seqid=0 2024-11-16T10:16:53,995 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/fe509def5cc74795a7e08e44058440ba as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/fe509def5cc74795a7e08e44058440ba 2024-11-16T10:16:53,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742396_1572 (size=31379) 2024-11-16T10:16:54,000 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 571a63535798b9dbe219ba8ef9231eab/B of 571a63535798b9dbe219ba8ef9231eab into fe509def5cc74795a7e08e44058440ba(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:54,000 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:54,000 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., storeName=571a63535798b9dbe219ba8ef9231eab/B, priority=12, startTime=1731752213534; duration=0sec 2024-11-16T10:16:54,000 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:54,000 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 571a63535798b9dbe219ba8ef9231eab:B 2024-11-16T10:16:54,000 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:16:54,001 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:16:54,001 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 571a63535798b9dbe219ba8ef9231eab/C is initiating minor compaction (all files) 2024-11-16T10:16:54,001 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 571a63535798b9dbe219ba8ef9231eab/C in TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:54,001 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/61d24a7ba099460aa2da37464e22aa31, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3b1396c3a2384b1f8068e4d2eeda7c92, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/53aa52d5d5574d0ca1f3d34d734740fb, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3a543f85da214aceb97e03f21f82b0a3] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp, totalSize=47.3 K 2024-11-16T10:16:54,003 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 61d24a7ba099460aa2da37464e22aa31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731752208462 2024-11-16T10:16:54,003 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b1396c3a2384b1f8068e4d2eeda7c92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731752209651 2024-11-16T10:16:54,003 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 53aa52d5d5574d0ca1f3d34d734740fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731752210333 2024-11-16T10:16:54,003 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a543f85da214aceb97e03f21f82b0a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731752212539 2024-11-16T10:16:54,011 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 571a63535798b9dbe219ba8ef9231eab#C#compaction#486 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:54,011 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/71b4766951d64ccabab7ffe61f8c5379 is 50, key is test_row_0/C:col10/1731752212559/Put/seqid=0 2024-11-16T10:16:54,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742397_1573 (size=12425) 2024-11-16T10:16:54,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:54,045 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161d47619cd3784c47a34be52300942b43_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161d47619cd3784c47a34be52300942b43_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:54,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/787f57b28f3842f387fd1fe79c3d2995, store: [table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:54,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/787f57b28f3842f387fd1fe79c3d2995 is 175, key is test_row_0/A:col10/1731752212691/Put/seqid=0 2024-11-16T10:16:54,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742398_1574 (size=31105) 2024-11-16T10:16:54,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752274164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:54,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752274164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:54,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T10:16:54,401 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/cca0b2726f8445008e7c205bdcb9862f as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/cca0b2726f8445008e7c205bdcb9862f 2024-11-16T10:16:54,405 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 571a63535798b9dbe219ba8ef9231eab/A of 571a63535798b9dbe219ba8ef9231eab into cca0b2726f8445008e7c205bdcb9862f(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:54,405 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:54,405 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., storeName=571a63535798b9dbe219ba8ef9231eab/A, priority=12, startTime=1731752213534; duration=0sec 2024-11-16T10:16:54,405 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:54,405 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 571a63535798b9dbe219ba8ef9231eab:A 2024-11-16T10:16:54,430 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/71b4766951d64ccabab7ffe61f8c5379 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/71b4766951d64ccabab7ffe61f8c5379 2024-11-16T10:16:54,435 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 571a63535798b9dbe219ba8ef9231eab/C of 571a63535798b9dbe219ba8ef9231eab into 71b4766951d64ccabab7ffe61f8c5379(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:54,435 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:54,435 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., storeName=571a63535798b9dbe219ba8ef9231eab/C, priority=12, startTime=1731752213534; duration=0sec 2024-11-16T10:16:54,436 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:54,436 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 571a63535798b9dbe219ba8ef9231eab:C 2024-11-16T10:16:54,449 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/787f57b28f3842f387fd1fe79c3d2995 2024-11-16T10:16:54,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/1be679472b8749a29cf84676c0e794c9 is 50, key is test_row_0/B:col10/1731752212691/Put/seqid=0 2024-11-16T10:16:54,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742399_1575 (size=12151) 2024-11-16T10:16:54,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752274474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:54,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:54,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:54,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:54,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752274474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:54,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:54,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:54,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:54,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752274844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:54,847 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in 571a63535798b9dbe219ba8ef9231eab of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:16:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:54,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752274854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:54,857 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in 571a63535798b9dbe219ba8ef9231eab of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:16:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:54,862 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/1be679472b8749a29cf84676c0e794c9 2024-11-16T10:16:54,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/19539025e7b646ba9fd928d2c504864c is 50, key is test_row_0/C:col10/1731752212691/Put/seqid=0 2024-11-16T10:16:54,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742400_1576 (size=12151) 2024-11-16T10:16:54,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752274984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:54,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:54,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:54,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:54,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752274993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:54,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:54,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:54,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:55,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43504 deadline: 1731752275094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:55,098 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_0' in 571a63535798b9dbe219ba8ef9231eab of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:16:55,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:55,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:55,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:55,270 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/19539025e7b646ba9fd928d2c504864c 2024-11-16T10:16:55,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/787f57b28f3842f387fd1fe79c3d2995 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/787f57b28f3842f387fd1fe79c3d2995 2024-11-16T10:16:55,276 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/787f57b28f3842f387fd1fe79c3d2995, entries=150, sequenceid=198, filesize=30.4 K 2024-11-16T10:16:55,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/1be679472b8749a29cf84676c0e794c9 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/1be679472b8749a29cf84676c0e794c9 2024-11-16T10:16:55,279 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/1be679472b8749a29cf84676c0e794c9, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T10:16:55,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/19539025e7b646ba9fd928d2c504864c as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/19539025e7b646ba9fd928d2c504864c 2024-11-16T10:16:55,282 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/19539025e7b646ba9fd928d2c504864c, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T10:16:55,283 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 571a63535798b9dbe219ba8ef9231eab in 1668ms, sequenceid=198, compaction requested=false 2024-11-16T10:16:55,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:55,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:55,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-16T10:16:55,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-16T10:16:55,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-16T10:16:55,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1270 sec 2024-11-16T10:16:55,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.1310 sec 2024-11-16T10:16:55,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T10:16:55,314 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:55,315 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:16:55,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-16T10:16:55,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T10:16:55,316 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:16:55,316 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:16:55,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:16:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T10:16:55,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T10:16:55,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:55,468 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 571a63535798b9dbe219ba8ef9231eab 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T10:16:55,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=A 2024-11-16T10:16:55,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:55,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=B 2024-11-16T10:16:55,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:55,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=C 2024-11-16T10:16:55,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:55,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168063d4ec90ba468a9b98d12ae4c36f24_571a63535798b9dbe219ba8ef9231eab is 50, key is test_row_0/A:col10/1731752213839/Put/seqid=0 2024-11-16T10:16:55,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742401_1577 (size=12304) 2024-11-16T10:16:55,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,479 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168063d4ec90ba468a9b98d12ae4c36f24_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168063d4ec90ba468a9b98d12ae4c36f24_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:55,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/d6741a1ec9bf4a0b8936ec4aaad447fc, store: [table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:55,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/d6741a1ec9bf4a0b8936ec4aaad447fc is 175, key is test_row_0/A:col10/1731752213839/Put/seqid=0 2024-11-16T10:16:55,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742402_1578 (size=31105) 2024-11-16T10:16:55,484 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/d6741a1ec9bf4a0b8936ec4aaad447fc 2024-11-16T10:16:55,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/c1ae31a137744cb98e8d43884cacad84 is 50, key is test_row_0/B:col10/1731752213839/Put/seqid=0 2024-11-16T10:16:55,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742403_1579 (size=12151) 2024-11-16T10:16:55,492 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/c1ae31a137744cb98e8d43884cacad84 2024-11-16T10:16:55,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/3862efda909e4d868dc52c4095d99822 is 50, key is test_row_0/C:col10/1731752213839/Put/seqid=0 2024-11-16T10:16:55,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742404_1580 (size=12151) 2024-11-16T10:16:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T10:16:55,901 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/3862efda909e4d868dc52c4095d99822 2024-11-16T10:16:55,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/d6741a1ec9bf4a0b8936ec4aaad447fc as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/d6741a1ec9bf4a0b8936ec4aaad447fc 2024-11-16T10:16:55,909 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/d6741a1ec9bf4a0b8936ec4aaad447fc, entries=150, sequenceid=211, filesize=30.4 K 2024-11-16T10:16:55,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/c1ae31a137744cb98e8d43884cacad84 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/c1ae31a137744cb98e8d43884cacad84 2024-11-16T10:16:55,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,912 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/c1ae31a137744cb98e8d43884cacad84, entries=150, sequenceid=211, filesize=11.9 K 2024-11-16T10:16:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/3862efda909e4d868dc52c4095d99822 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3862efda909e4d868dc52c4095d99822 2024-11-16T10:16:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,916 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3862efda909e4d868dc52c4095d99822, entries=150, sequenceid=211, filesize=11.9 K 2024-11-16T10:16:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,917 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 571a63535798b9dbe219ba8ef9231eab in 449ms, sequenceid=211, compaction requested=true 2024-11-16T10:16:55,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:55,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-16T10:16:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-16T10:16:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-16T10:16:55,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 602 msec 2024-11-16T10:16:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 606 msec 2024-11-16T10:16:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T10:16:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,944 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,945 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:16:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-16T10:16:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,947 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:16:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,947 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:16:55,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:16:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T10:16:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:56,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 571a63535798b9dbe219ba8ef9231eab 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T10:16:56,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=A 2024-11-16T10:16:56,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:56,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=B 2024-11-16T10:16:56,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:56,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=C 2024-11-16T10:16:56,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b831d6aaf60c443aa6f065e7bcbb5ff8_571a63535798b9dbe219ba8ef9231eab is 50, key is test_row_0/A:col10/1731752216042/Put/seqid=0 2024-11-16T10:16:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T10:16:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742405_1581 (size=22268) 2024-11-16T10:16:56,078 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:56,085 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b831d6aaf60c443aa6f065e7bcbb5ff8_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b831d6aaf60c443aa6f065e7bcbb5ff8_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:56,086 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/100e99f14a554fd98fc7a920044d73f8, store: [table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:56,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/100e99f14a554fd98fc7a920044d73f8 is 175, key is test_row_0/A:col10/1731752216042/Put/seqid=0 2024-11-16T10:16:56,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742406_1582 (size=65673) 2024-11-16T10:16:56,094 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=221, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/100e99f14a554fd98fc7a920044d73f8 2024-11-16T10:16:56,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/ff753a007f914ac79d273a8dcdfbf08c is 50, key is test_row_0/B:col10/1731752216042/Put/seqid=0 2024-11-16T10:16:56,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T10:16:56,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:56,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:56,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:56,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:56,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:56,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752216099, max timestamp=1731752216118Num uniques 859; 2024-11-16T10:16:56,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:16:56,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752216099, max timestamp=1731752216118Num uniques 859; 2024-11-16T10:16:56,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:16:56,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731752216099, max timestamp=1731752216118Num uniques 859; 2024-11-16T10:16:56,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b16ff38458c1:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T10:16:56,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742407_1583 (size=12151) 2024-11-16T10:16:56,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752276178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:56,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:56,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:56,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752276185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:56,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T10:16:56,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:56,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:56,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:56,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:56,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:56,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T10:16:56,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752276294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:56,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752276294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:56,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T10:16:56,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:56,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:56,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:56,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:56,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:56,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:56,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752276504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:56,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:56,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752276504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:56,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:56,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:56,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:56,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:56,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:56,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/ff753a007f914ac79d273a8dcdfbf08c 2024-11-16T10:16:56,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/0160311842f74c7099f49caaa4b57c7b is 50, key is test_row_0/C:col10/1731752216042/Put/seqid=0 2024-11-16T10:16:56,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742408_1584 (size=12151) 2024-11-16T10:16:56,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T10:16:56,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:56,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:56,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:56,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:56,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:56,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T10:16:56,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T10:16:56,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:56,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:56,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:56,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:56,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:56,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752276813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:56,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752276814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:56,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:56,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752276854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:56,858 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in 571a63535798b9dbe219ba8ef9231eab of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:16:56,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:56,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:56,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:56,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T10:16:56,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:56,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:56,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:56,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:56,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:56,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:56,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752276874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:56,877 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in 571a63535798b9dbe219ba8ef9231eab of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:16:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:56,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/0160311842f74c7099f49caaa4b57c7b 2024-11-16T10:16:56,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/100e99f14a554fd98fc7a920044d73f8 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/100e99f14a554fd98fc7a920044d73f8 2024-11-16T10:16:56,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/100e99f14a554fd98fc7a920044d73f8, entries=350, sequenceid=221, filesize=64.1 K 2024-11-16T10:16:56,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/ff753a007f914ac79d273a8dcdfbf08c as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/ff753a007f914ac79d273a8dcdfbf08c 2024-11-16T10:16:56,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/ff753a007f914ac79d273a8dcdfbf08c, entries=150, sequenceid=221, filesize=11.9 K 2024-11-16T10:16:56,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/0160311842f74c7099f49caaa4b57c7b as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/0160311842f74c7099f49caaa4b57c7b 2024-11-16T10:16:56,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/0160311842f74c7099f49caaa4b57c7b, entries=150, sequenceid=221, filesize=11.9 K 2024-11-16T10:16:56,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 571a63535798b9dbe219ba8ef9231eab in 920ms, sequenceid=221, compaction requested=true 2024-11-16T10:16:56,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:56,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 571a63535798b9dbe219ba8ef9231eab:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:16:56,964 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:16:56,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:56,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 571a63535798b9dbe219ba8ef9231eab:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:16:56,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:56,964 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:16:56,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 571a63535798b9dbe219ba8ef9231eab:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:16:56,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:56,969 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:16:56,969 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 159262 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:16:56,969 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 571a63535798b9dbe219ba8ef9231eab/B is initiating minor compaction (all files) 2024-11-16T10:16:56,969 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 571a63535798b9dbe219ba8ef9231eab/A is initiating minor compaction (all files) 2024-11-16T10:16:56,969 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 571a63535798b9dbe219ba8ef9231eab/B in TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:56,969 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 571a63535798b9dbe219ba8ef9231eab/A in TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:56,969 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/fe509def5cc74795a7e08e44058440ba, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/1be679472b8749a29cf84676c0e794c9, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/c1ae31a137744cb98e8d43884cacad84, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/ff753a007f914ac79d273a8dcdfbf08c] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp, totalSize=47.7 K 2024-11-16T10:16:56,969 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/cca0b2726f8445008e7c205bdcb9862f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/787f57b28f3842f387fd1fe79c3d2995, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/d6741a1ec9bf4a0b8936ec4aaad447fc, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/100e99f14a554fd98fc7a920044d73f8] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp, totalSize=155.5 K 2024-11-16T10:16:56,969 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:56,969 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. files: [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/cca0b2726f8445008e7c205bdcb9862f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/787f57b28f3842f387fd1fe79c3d2995, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/d6741a1ec9bf4a0b8936ec4aaad447fc, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/100e99f14a554fd98fc7a920044d73f8] 2024-11-16T10:16:56,970 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting fe509def5cc74795a7e08e44058440ba, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731752212539 2024-11-16T10:16:56,970 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting cca0b2726f8445008e7c205bdcb9862f, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731752212539 2024-11-16T10:16:56,970 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 1be679472b8749a29cf84676c0e794c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731752212675 2024-11-16T10:16:56,970 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 787f57b28f3842f387fd1fe79c3d2995, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731752212675 2024-11-16T10:16:56,970 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting c1ae31a137744cb98e8d43884cacad84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731752213828 2024-11-16T10:16:56,970 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6741a1ec9bf4a0b8936ec4aaad447fc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731752213828 2024-11-16T10:16:56,970 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 100e99f14a554fd98fc7a920044d73f8, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731752216010 2024-11-16T10:16:56,970 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting ff753a007f914ac79d273a8dcdfbf08c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731752216034 2024-11-16T10:16:56,977 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:56,978 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111633d0e68630204343843ca42dad3e2c17_571a63535798b9dbe219ba8ef9231eab store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:56,980 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 571a63535798b9dbe219ba8ef9231eab#B#compaction#496 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:56,980 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/7161b6ea279c42fd8c8704d98c0ba1d8 is 50, key is test_row_0/B:col10/1731752216042/Put/seqid=0 2024-11-16T10:16:56,981 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111633d0e68630204343843ca42dad3e2c17_571a63535798b9dbe219ba8ef9231eab, store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:56,981 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111633d0e68630204343843ca42dad3e2c17_571a63535798b9dbe219ba8ef9231eab because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:56,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742409_1585 (size=12561) 2024-11-16T10:16:56,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742410_1586 (size=4469) 2024-11-16T10:16:57,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T10:16:57,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:57,016 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 571a63535798b9dbe219ba8ef9231eab 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-16T10:16:57,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=A 2024-11-16T10:16:57,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:57,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=B 2024-11-16T10:16:57,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:57,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=C 2024-11-16T10:16:57,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T10:16:57,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c90bf5b62e7246bd9c733bb68cf51786_571a63535798b9dbe219ba8ef9231eab is 50, key is test_row_0/A:col10/1731752216183/Put/seqid=0 2024-11-16T10:16:57,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742411_1587 (size=12354) 2024-11-16T10:16:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T10:16:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:57,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:57,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752277340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:57,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752277343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:57,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:57,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:57,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:57,399 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 571a63535798b9dbe219ba8ef9231eab#A#compaction#495 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:57,399 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/74b601071c96401791cd6fa7a2ced536 is 175, key is test_row_0/A:col10/1731752216042/Put/seqid=0 2024-11-16T10:16:57,401 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/7161b6ea279c42fd8c8704d98c0ba1d8 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/7161b6ea279c42fd8c8704d98c0ba1d8 2024-11-16T10:16:57,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742412_1588 (size=31515) 2024-11-16T10:16:57,405 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 571a63535798b9dbe219ba8ef9231eab/B of 571a63535798b9dbe219ba8ef9231eab into 7161b6ea279c42fd8c8704d98c0ba1d8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:57,405 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:57,405 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., storeName=571a63535798b9dbe219ba8ef9231eab/B, priority=12, startTime=1731752216964; duration=0sec 2024-11-16T10:16:57,405 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:57,405 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 571a63535798b9dbe219ba8ef9231eab:B 2024-11-16T10:16:57,405 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:16:57,406 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:16:57,406 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 571a63535798b9dbe219ba8ef9231eab/C is initiating minor compaction (all files) 2024-11-16T10:16:57,406 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 571a63535798b9dbe219ba8ef9231eab/C in TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:57,406 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/71b4766951d64ccabab7ffe61f8c5379, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/19539025e7b646ba9fd928d2c504864c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3862efda909e4d868dc52c4095d99822, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/0160311842f74c7099f49caaa4b57c7b] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp, totalSize=47.7 K 2024-11-16T10:16:57,406 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 71b4766951d64ccabab7ffe61f8c5379, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731752212539 2024-11-16T10:16:57,406 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 19539025e7b646ba9fd928d2c504864c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731752212675 2024-11-16T10:16:57,407 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 3862efda909e4d868dc52c4095d99822, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731752213828 2024-11-16T10:16:57,407 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 0160311842f74c7099f49caaa4b57c7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731752216034 2024-11-16T10:16:57,413 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 571a63535798b9dbe219ba8ef9231eab#C#compaction#498 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:57,414 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/65c0aae7a4624513a1177cc07c252f43 is 50, key is test_row_0/C:col10/1731752216042/Put/seqid=0 2024-11-16T10:16:57,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742413_1589 (size=12561) 2024-11-16T10:16:57,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:57,433 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c90bf5b62e7246bd9c733bb68cf51786_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c90bf5b62e7246bd9c733bb68cf51786_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:57,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/d97fc00ad2df4bf0966cbfa6b6b9b267, store: [table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:57,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/d97fc00ad2df4bf0966cbfa6b6b9b267 is 175, key is test_row_0/A:col10/1731752216183/Put/seqid=0 2024-11-16T10:16:57,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742414_1590 (size=31155) 2024-11-16T10:16:57,453 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/d97fc00ad2df4bf0966cbfa6b6b9b267 2024-11-16T10:16:57,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752277454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:57,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752277454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:57,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/979a34ca31a44012b291dd9846658e50 is 50, key is test_row_0/B:col10/1731752216183/Put/seqid=0 2024-11-16T10:16:57,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742415_1591 (size=12201) 2024-11-16T10:16:57,477 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/979a34ca31a44012b291dd9846658e50 2024-11-16T10:16:57,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/4cca84c8ec274381a6c222d9bcfdc470 is 50, key is test_row_0/C:col10/1731752216183/Put/seqid=0 2024-11-16T10:16:57,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742416_1592 (size=12201) 2024-11-16T10:16:57,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:57,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752277664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:57,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:57,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752277664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:57,806 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/74b601071c96401791cd6fa7a2ced536 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/74b601071c96401791cd6fa7a2ced536 2024-11-16T10:16:57,809 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 571a63535798b9dbe219ba8ef9231eab/A of 571a63535798b9dbe219ba8ef9231eab into 74b601071c96401791cd6fa7a2ced536(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:57,809 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:57,809 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., storeName=571a63535798b9dbe219ba8ef9231eab/A, priority=12, startTime=1731752216964; duration=0sec 2024-11-16T10:16:57,809 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:57,809 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 571a63535798b9dbe219ba8ef9231eab:A 2024-11-16T10:16:57,820 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/65c0aae7a4624513a1177cc07c252f43 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/65c0aae7a4624513a1177cc07c252f43 2024-11-16T10:16:57,823 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 571a63535798b9dbe219ba8ef9231eab/C of 571a63535798b9dbe219ba8ef9231eab into 65c0aae7a4624513a1177cc07c252f43(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:57,823 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:57,823 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., storeName=571a63535798b9dbe219ba8ef9231eab/C, priority=12, startTime=1731752216964; duration=0sec 2024-11-16T10:16:57,823 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:57,823 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 571a63535798b9dbe219ba8ef9231eab:C 2024-11-16T10:16:57,898 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/4cca84c8ec274381a6c222d9bcfdc470 2024-11-16T10:16:57,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/d97fc00ad2df4bf0966cbfa6b6b9b267 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/d97fc00ad2df4bf0966cbfa6b6b9b267 2024-11-16T10:16:57,903 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/d97fc00ad2df4bf0966cbfa6b6b9b267, entries=150, sequenceid=258, filesize=30.4 K 2024-11-16T10:16:57,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/979a34ca31a44012b291dd9846658e50 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/979a34ca31a44012b291dd9846658e50 2024-11-16T10:16:57,906 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/979a34ca31a44012b291dd9846658e50, entries=150, sequenceid=258, filesize=11.9 K 2024-11-16T10:16:57,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/4cca84c8ec274381a6c222d9bcfdc470 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/4cca84c8ec274381a6c222d9bcfdc470 2024-11-16T10:16:57,910 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/4cca84c8ec274381a6c222d9bcfdc470, entries=150, sequenceid=258, filesize=11.9 K 2024-11-16T10:16:57,910 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 571a63535798b9dbe219ba8ef9231eab in 894ms, sequenceid=258, compaction requested=false 2024-11-16T10:16:57,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:57,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:57,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-16T10:16:57,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-16T10:16:57,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-16T10:16:57,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9650 sec 2024-11-16T10:16:57,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.9690 sec 2024-11-16T10:16:57,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:57,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 571a63535798b9dbe219ba8ef9231eab 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T10:16:57,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=A 2024-11-16T10:16:57,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:57,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=B 2024-11-16T10:16:57,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:57,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=C 2024-11-16T10:16:57,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:57,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111619fa56339aa243b28b7274af47a9b0da_571a63535798b9dbe219ba8ef9231eab is 50, key is test_row_0/A:col10/1731752217976/Put/seqid=0 2024-11-16T10:16:57,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742417_1593 (size=17534) 2024-11-16T10:16:57,987 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:57,991 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111619fa56339aa243b28b7274af47a9b0da_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111619fa56339aa243b28b7274af47a9b0da_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:57,992 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/418031ada2e448ae812761f04da32ae2, store: [table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:57,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/418031ada2e448ae812761f04da32ae2 is 175, key is test_row_0/A:col10/1731752217976/Put/seqid=0 2024-11-16T10:16:57,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742418_1594 (size=48639) 2024-11-16T10:16:57,998 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/418031ada2e448ae812761f04da32ae2 2024-11-16T10:16:58,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/823d9995fb1c4325a9461f2c7d9a8037 is 50, key is test_row_0/B:col10/1731752217976/Put/seqid=0 2024-11-16T10:16:58,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742419_1595 (size=12301) 2024-11-16T10:16:58,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:58,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752278024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:58,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:58,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752278030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:58,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:58,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:58,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:58,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T10:16:58,094 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:16:58,095 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:16:58,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-16T10:16:58,096 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:16:58,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T10:16:58,097 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:16:58,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:16:58,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:58,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752278133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:58,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:58,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:58,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:58,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:58,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752278144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:58,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:58,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:58,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:58,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T10:16:58,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T10:16:58,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:58,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:58,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:58,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:58,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:58,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:58,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752278345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:58,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:58,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:58,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:58,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752278354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:58,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:58,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:58,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:58,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T10:16:58,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:58,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:58,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:58,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:58,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:16:58,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/823d9995fb1c4325a9461f2c7d9a8037 2024-11-16T10:16:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T10:16:58,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/be684bf28eef4ca18071ea347cfd61f9 is 50, key is test_row_0/C:col10/1731752217976/Put/seqid=0 2024-11-16T10:16:58,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742420_1596 (size=12301) 2024-11-16T10:16:58,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/be684bf28eef4ca18071ea347cfd61f9 2024-11-16T10:16:58,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/418031ada2e448ae812761f04da32ae2 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/418031ada2e448ae812761f04da32ae2 2024-11-16T10:16:58,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/418031ada2e448ae812761f04da32ae2, entries=250, sequenceid=273, filesize=47.5 K 2024-11-16T10:16:58,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/823d9995fb1c4325a9461f2c7d9a8037 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/823d9995fb1c4325a9461f2c7d9a8037 2024-11-16T10:16:58,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/823d9995fb1c4325a9461f2c7d9a8037, entries=150, sequenceid=273, filesize=12.0 K 2024-11-16T10:16:58,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/be684bf28eef4ca18071ea347cfd61f9 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/be684bf28eef4ca18071ea347cfd61f9 2024-11-16T10:16:58,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/be684bf28eef4ca18071ea347cfd61f9, entries=150, sequenceid=273, filesize=12.0 K 2024-11-16T10:16:58,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 571a63535798b9dbe219ba8ef9231eab in 488ms, sequenceid=273, compaction requested=true 2024-11-16T10:16:58,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:58,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 571a63535798b9dbe219ba8ef9231eab:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:16:58,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:58,465 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:58,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 571a63535798b9dbe219ba8ef9231eab:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:16:58,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:58,465 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:58,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 571a63535798b9dbe219ba8ef9231eab:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:16:58,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:58,466 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:58,466 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37063 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:58,466 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 571a63535798b9dbe219ba8ef9231eab/A is initiating minor compaction (all files) 2024-11-16T10:16:58,466 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 571a63535798b9dbe219ba8ef9231eab/B is initiating minor compaction (all files) 2024-11-16T10:16:58,466 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 571a63535798b9dbe219ba8ef9231eab/A in TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:58,466 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 571a63535798b9dbe219ba8ef9231eab/B in TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:58,466 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/74b601071c96401791cd6fa7a2ced536, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/d97fc00ad2df4bf0966cbfa6b6b9b267, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/418031ada2e448ae812761f04da32ae2] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp, totalSize=108.7 K 2024-11-16T10:16:58,466 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/7161b6ea279c42fd8c8704d98c0ba1d8, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/979a34ca31a44012b291dd9846658e50, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/823d9995fb1c4325a9461f2c7d9a8037] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp, totalSize=36.2 K 2024-11-16T10:16:58,466 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:58,466 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. files: [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/74b601071c96401791cd6fa7a2ced536, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/d97fc00ad2df4bf0966cbfa6b6b9b267, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/418031ada2e448ae812761f04da32ae2] 2024-11-16T10:16:58,466 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 7161b6ea279c42fd8c8704d98c0ba1d8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731752216034 2024-11-16T10:16:58,466 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74b601071c96401791cd6fa7a2ced536, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731752216034 2024-11-16T10:16:58,467 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 979a34ca31a44012b291dd9846658e50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731752216176 2024-11-16T10:16:58,467 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting d97fc00ad2df4bf0966cbfa6b6b9b267, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731752216176 2024-11-16T10:16:58,467 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 823d9995fb1c4325a9461f2c7d9a8037, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731752217339 2024-11-16T10:16:58,467 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 418031ada2e448ae812761f04da32ae2, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731752217332 2024-11-16T10:16:58,472 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:58,474 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 571a63535798b9dbe219ba8ef9231eab#B#compaction#505 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:58,475 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/24770de74b6e417a90b5dbc7c503c4b1 is 50, key is test_row_0/B:col10/1731752217976/Put/seqid=0 2024-11-16T10:16:58,480 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116bbb074cc66b341c69089b489ff5c74a7_571a63535798b9dbe219ba8ef9231eab store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:58,482 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116bbb074cc66b341c69089b489ff5c74a7_571a63535798b9dbe219ba8ef9231eab, store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:58,482 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116bbb074cc66b341c69089b489ff5c74a7_571a63535798b9dbe219ba8ef9231eab because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:58,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742421_1597 (size=12813) 2024-11-16T10:16:58,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742422_1598 (size=4469) 2024-11-16T10:16:58,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T10:16:58,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:58,557 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 571a63535798b9dbe219ba8ef9231eab 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T10:16:58,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=A 2024-11-16T10:16:58,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:58,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=B 2024-11-16T10:16:58,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:58,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=C 2024-11-16T10:16:58,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:58,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111686aa0f9aabaa4452a82beca5d9e03535_571a63535798b9dbe219ba8ef9231eab is 50, key is test_row_0/A:col10/1731752218023/Put/seqid=0 2024-11-16T10:16:58,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742423_1599 (size=12454) 2024-11-16T10:16:58,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:58,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:16:58,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752278686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:58,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:58,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:58,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:58,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752278691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T10:16:58,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752278793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:58,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:58,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:58,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:58,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752278804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:58,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:58,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:58,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:58,888 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 571a63535798b9dbe219ba8ef9231eab#A#compaction#504 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:58,888 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/a005469487354ec59ed800778d6e4219 is 175, key is test_row_0/A:col10/1731752217976/Put/seqid=0 2024-11-16T10:16:58,893 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/24770de74b6e417a90b5dbc7c503c4b1 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/24770de74b6e417a90b5dbc7c503c4b1 2024-11-16T10:16:58,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742424_1600 (size=31767) 2024-11-16T10:16:58,898 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 571a63535798b9dbe219ba8ef9231eab/B of 571a63535798b9dbe219ba8ef9231eab into 24770de74b6e417a90b5dbc7c503c4b1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:58,898 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:58,898 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., storeName=571a63535798b9dbe219ba8ef9231eab/B, priority=13, startTime=1731752218465; duration=0sec 2024-11-16T10:16:58,898 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:16:58,898 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 571a63535798b9dbe219ba8ef9231eab:B 2024-11-16T10:16:58,898 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:16:58,899 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37063 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:16:58,899 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 571a63535798b9dbe219ba8ef9231eab/C is initiating minor compaction (all files) 2024-11-16T10:16:58,899 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 571a63535798b9dbe219ba8ef9231eab/C in TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:58,899 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/65c0aae7a4624513a1177cc07c252f43, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/4cca84c8ec274381a6c222d9bcfdc470, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/be684bf28eef4ca18071ea347cfd61f9] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp, totalSize=36.2 K 2024-11-16T10:16:58,900 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 65c0aae7a4624513a1177cc07c252f43, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731752216034 2024-11-16T10:16:58,900 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cca84c8ec274381a6c222d9bcfdc470, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731752216176 2024-11-16T10:16:58,900 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/a005469487354ec59ed800778d6e4219 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/a005469487354ec59ed800778d6e4219 2024-11-16T10:16:58,901 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting be684bf28eef4ca18071ea347cfd61f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731752217339 2024-11-16T10:16:58,905 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 571a63535798b9dbe219ba8ef9231eab/A of 571a63535798b9dbe219ba8ef9231eab into a005469487354ec59ed800778d6e4219(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:58,905 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:58,905 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., storeName=571a63535798b9dbe219ba8ef9231eab/A, priority=13, startTime=1731752218465; duration=0sec 2024-11-16T10:16:58,906 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:58,906 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 571a63535798b9dbe219ba8ef9231eab:A 2024-11-16T10:16:58,910 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 571a63535798b9dbe219ba8ef9231eab#C#compaction#507 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:16:58,911 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/fb9fcbb0629a4e1994bb274ccf64aa8f is 50, key is test_row_0/C:col10/1731752217976/Put/seqid=0 2024-11-16T10:16:58,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742425_1601 (size=12813) 2024-11-16T10:16:58,918 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/fb9fcbb0629a4e1994bb274ccf64aa8f as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/fb9fcbb0629a4e1994bb274ccf64aa8f 2024-11-16T10:16:58,922 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 571a63535798b9dbe219ba8ef9231eab/C of 571a63535798b9dbe219ba8ef9231eab into fb9fcbb0629a4e1994bb274ccf64aa8f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:16:58,922 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:58,922 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., storeName=571a63535798b9dbe219ba8ef9231eab/C, priority=13, startTime=1731752218465; duration=0sec 2024-11-16T10:16:58,922 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:16:58,922 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 571a63535798b9dbe219ba8ef9231eab:C 2024-11-16T10:16:58,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:16:58,982 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111686aa0f9aabaa4452a82beca5d9e03535_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111686aa0f9aabaa4452a82beca5d9e03535_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:58,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/5e3ab2186f624faaadf5cfeb75c5700f, store: [table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:16:58,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/5e3ab2186f624faaadf5cfeb75c5700f is 175, key is test_row_0/A:col10/1731752218023/Put/seqid=0 2024-11-16T10:16:58,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742426_1602 (size=31255) 2024-11-16T10:16:59,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:59,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752279010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:59,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:59,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:59,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:59,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:59,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752279014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:59,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:59,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:59,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:59,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T10:16:59,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:59,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752279324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:59,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752279324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:59,395 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/5e3ab2186f624faaadf5cfeb75c5700f 2024-11-16T10:16:59,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/e53da9d873324413bf3f1d29fdc31154 is 50, key is test_row_0/B:col10/1731752218023/Put/seqid=0 2024-11-16T10:16:59,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742427_1603 (size=12301) 2024-11-16T10:16:59,806 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/e53da9d873324413bf3f1d29fdc31154 2024-11-16T10:16:59,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/2a2a2bfd102844368a6733c4fc5eb104 is 50, key is test_row_0/C:col10/1731752218023/Put/seqid=0 2024-11-16T10:16:59,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742428_1604 (size=12301) 2024-11-16T10:16:59,815 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/2a2a2bfd102844368a6733c4fc5eb104 2024-11-16T10:16:59,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/5e3ab2186f624faaadf5cfeb75c5700f as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/5e3ab2186f624faaadf5cfeb75c5700f 2024-11-16T10:16:59,821 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/5e3ab2186f624faaadf5cfeb75c5700f, entries=150, sequenceid=293, filesize=30.5 K 2024-11-16T10:16:59,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/e53da9d873324413bf3f1d29fdc31154 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/e53da9d873324413bf3f1d29fdc31154 2024-11-16T10:16:59,824 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/e53da9d873324413bf3f1d29fdc31154, entries=150, sequenceid=293, filesize=12.0 K 2024-11-16T10:16:59,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/2a2a2bfd102844368a6733c4fc5eb104 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/2a2a2bfd102844368a6733c4fc5eb104 2024-11-16T10:16:59,827 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/2a2a2bfd102844368a6733c4fc5eb104, entries=150, sequenceid=293, filesize=12.0 K 2024-11-16T10:16:59,828 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 571a63535798b9dbe219ba8ef9231eab in 1271ms, sequenceid=293, compaction requested=false 2024-11-16T10:16:59,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:16:59,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:16:59,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-16T10:16:59,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-16T10:16:59,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-16T10:16:59,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7320 sec 2024-11-16T10:16:59,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.7360 sec 2024-11-16T10:16:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:16:59,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 571a63535798b9dbe219ba8ef9231eab 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T10:16:59,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=A 2024-11-16T10:16:59,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:59,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=B 2024-11-16T10:16:59,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:59,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=C 2024-11-16T10:16:59,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:16:59,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111677ef88ada296421bbf21bc46595cc0b6_571a63535798b9dbe219ba8ef9231eab is 50, key is test_row_0/A:col10/1731752218690/Put/seqid=0 2024-11-16T10:16:59,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742429_1605 (size=17534) 2024-11-16T10:16:59,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:59,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752279877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:59,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:59,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:59,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:59,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752279882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:59,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:59,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:59,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:59,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752279994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:59,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:16:59,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752279994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:16:59,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:59,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:16:59,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:59,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:16:59,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:00,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752280204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:00,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:00,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:00,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:00,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752280204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:00,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:00,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:00,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:00,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T10:17:00,244 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:17:00,245 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:17:00,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-16T10:17:00,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T10:17:00,246 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:17:00,248 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:17:00,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:17:00,254 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:17:00,257 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111677ef88ada296421bbf21bc46595cc0b6_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111677ef88ada296421bbf21bc46595cc0b6_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:00,258 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/93a8dbdcf3dd44cba1fc16dd5957ae0d, store: [table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:17:00,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/93a8dbdcf3dd44cba1fc16dd5957ae0d is 175, key is test_row_0/A:col10/1731752218690/Put/seqid=0 2024-11-16T10:17:00,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742430_1606 (size=48639) 2024-11-16T10:17:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T10:17:00,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T10:17:00,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:00,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:17:00,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:00,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:00,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:00,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:00,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752280514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:00,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752280514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:00,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T10:17:00,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:00,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:17:00,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:00,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:00,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:00,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T10:17:00,663 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=310, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/93a8dbdcf3dd44cba1fc16dd5957ae0d 2024-11-16T10:17:00,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/5497c416c16b466e8814bf4e5d2470a1 is 50, key is test_row_0/B:col10/1731752218690/Put/seqid=0 2024-11-16T10:17:00,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742431_1607 (size=12301) 2024-11-16T10:17:00,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/5497c416c16b466e8814bf4e5d2470a1 2024-11-16T10:17:00,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/e2e97daeaa1d4babb424b4b756f8bd36 is 50, key is test_row_0/C:col10/1731752218690/Put/seqid=0 2024-11-16T10:17:00,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742432_1608 (size=12301) 2024-11-16T10:17:00,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T10:17:00,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:00,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:17:00,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:00,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:00,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:00,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T10:17:00,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:00,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:17:00,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:00,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:00,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:00,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:00,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T10:17:00,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1731752280894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:00,896 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in 571a63535798b9dbe219ba8ef9231eab of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:17:00,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:00,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:00,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:00,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43482 deadline: 1731752280894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:00,897 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_1' in 571a63535798b9dbe219ba8ef9231eab of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:17:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:01,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T10:17:01,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:01,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:17:01,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:01,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:01,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:01,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752281024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:01,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752281024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:01,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:01,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:01,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:01,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:01,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:01,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:01,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/e2e97daeaa1d4babb424b4b756f8bd36 2024-11-16T10:17:01,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/93a8dbdcf3dd44cba1fc16dd5957ae0d as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/93a8dbdcf3dd44cba1fc16dd5957ae0d 2024-11-16T10:17:01,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/93a8dbdcf3dd44cba1fc16dd5957ae0d, entries=250, sequenceid=310, filesize=47.5 K 2024-11-16T10:17:01,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/5497c416c16b466e8814bf4e5d2470a1 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/5497c416c16b466e8814bf4e5d2470a1 2024-11-16T10:17:01,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/5497c416c16b466e8814bf4e5d2470a1, entries=150, sequenceid=310, filesize=12.0 K 2024-11-16T10:17:01,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/e2e97daeaa1d4babb424b4b756f8bd36 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/e2e97daeaa1d4babb424b4b756f8bd36 2024-11-16T10:17:01,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/e2e97daeaa1d4babb424b4b756f8bd36, entries=150, sequenceid=310, filesize=12.0 K 2024-11-16T10:17:01,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 571a63535798b9dbe219ba8ef9231eab in 1279ms, sequenceid=310, compaction requested=true 2024-11-16T10:17:01,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:17:01,119 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:17:01,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 571a63535798b9dbe219ba8ef9231eab:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:17:01,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:17:01,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 571a63535798b9dbe219ba8ef9231eab:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:17:01,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:17:01,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 571a63535798b9dbe219ba8ef9231eab:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:17:01,120 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:17:01,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:17:01,120 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:17:01,120 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 571a63535798b9dbe219ba8ef9231eab/A is initiating minor compaction (all files) 2024-11-16T10:17:01,120 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 571a63535798b9dbe219ba8ef9231eab/A in TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:01,121 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/a005469487354ec59ed800778d6e4219, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/5e3ab2186f624faaadf5cfeb75c5700f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/93a8dbdcf3dd44cba1fc16dd5957ae0d] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp, totalSize=109.0 K 2024-11-16T10:17:01,121 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:01,121 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. files: [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/a005469487354ec59ed800778d6e4219, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/5e3ab2186f624faaadf5cfeb75c5700f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/93a8dbdcf3dd44cba1fc16dd5957ae0d] 2024-11-16T10:17:01,121 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:17:01,121 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 571a63535798b9dbe219ba8ef9231eab/B is initiating minor compaction (all files) 2024-11-16T10:17:01,121 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 571a63535798b9dbe219ba8ef9231eab/B in TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:01,121 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/24770de74b6e417a90b5dbc7c503c4b1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/e53da9d873324413bf3f1d29fdc31154, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/5497c416c16b466e8814bf4e5d2470a1] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp, totalSize=36.5 K 2024-11-16T10:17:01,121 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting a005469487354ec59ed800778d6e4219, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731752217339 2024-11-16T10:17:01,121 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 24770de74b6e417a90b5dbc7c503c4b1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731752217339 2024-11-16T10:17:01,121 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e3ab2186f624faaadf5cfeb75c5700f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731752218001 2024-11-16T10:17:01,121 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting e53da9d873324413bf3f1d29fdc31154, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731752218001 2024-11-16T10:17:01,122 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93a8dbdcf3dd44cba1fc16dd5957ae0d, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731752218684 2024-11-16T10:17:01,124 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 5497c416c16b466e8814bf4e5d2470a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731752218690 2024-11-16T10:17:01,130 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:17:01,134 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 571a63535798b9dbe219ba8ef9231eab#B#compaction#514 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:17:01,134 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/c5891d195a9f48cbbd170b654af86b2a is 50, key is test_row_0/B:col10/1731752218690/Put/seqid=0 2024-11-16T10:17:01,135 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411165cc0f1be0abc4e68829dde2b415cc5b9_571a63535798b9dbe219ba8ef9231eab store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:17:01,137 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411165cc0f1be0abc4e68829dde2b415cc5b9_571a63535798b9dbe219ba8ef9231eab, store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:17:01,137 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165cc0f1be0abc4e68829dde2b415cc5b9_571a63535798b9dbe219ba8ef9231eab because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:17:01,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742433_1609 (size=12915) 2024-11-16T10:17:01,147 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/c5891d195a9f48cbbd170b654af86b2a as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/c5891d195a9f48cbbd170b654af86b2a 2024-11-16T10:17:01,151 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 571a63535798b9dbe219ba8ef9231eab/B of 571a63535798b9dbe219ba8ef9231eab into c5891d195a9f48cbbd170b654af86b2a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:17:01,151 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:17:01,151 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., storeName=571a63535798b9dbe219ba8ef9231eab/B, priority=13, startTime=1731752221120; duration=0sec 2024-11-16T10:17:01,151 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:17:01,151 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 571a63535798b9dbe219ba8ef9231eab:B 2024-11-16T10:17:01,151 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:17:01,152 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:17:01,152 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 571a63535798b9dbe219ba8ef9231eab/C is initiating minor compaction (all files) 2024-11-16T10:17:01,152 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 571a63535798b9dbe219ba8ef9231eab/C in TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:01,152 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/fb9fcbb0629a4e1994bb274ccf64aa8f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/2a2a2bfd102844368a6733c4fc5eb104, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/e2e97daeaa1d4babb424b4b756f8bd36] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp, totalSize=36.5 K 2024-11-16T10:17:01,152 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting fb9fcbb0629a4e1994bb274ccf64aa8f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731752217339 2024-11-16T10:17:01,152 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a2a2bfd102844368a6733c4fc5eb104, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731752218001 2024-11-16T10:17:01,153 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting e2e97daeaa1d4babb424b4b756f8bd36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731752218690 2024-11-16T10:17:01,158 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 571a63535798b9dbe219ba8ef9231eab#C#compaction#515 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:17:01,158 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/884cfd2256b64e2599138fc85f5b79fa is 50, key is test_row_0/C:col10/1731752218690/Put/seqid=0 2024-11-16T10:17:01,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742434_1610 (size=4469) 2024-11-16T10:17:01,161 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 571a63535798b9dbe219ba8ef9231eab#A#compaction#513 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:17:01,161 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/9ba01a9319ca42a2a07c951bd91692cf is 175, key is test_row_0/A:col10/1731752218690/Put/seqid=0 2024-11-16T10:17:01,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T10:17:01,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:01,167 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 571a63535798b9dbe219ba8ef9231eab 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-16T10:17:01,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=A 2024-11-16T10:17:01,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:17:01,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=B 2024-11-16T10:17:01,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:17:01,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=C 2024-11-16T10:17:01,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:17:01,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742436_1612 (size=31869) 2024-11-16T10:17:01,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742435_1611 (size=12915) 2024-11-16T10:17:01,181 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/9ba01a9319ca42a2a07c951bd91692cf as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/9ba01a9319ca42a2a07c951bd91692cf 2024-11-16T10:17:01,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ea44957cf2b94c9d8389982a3d8f5446_571a63535798b9dbe219ba8ef9231eab is 50, key is test_row_0/A:col10/1731752219869/Put/seqid=0 2024-11-16T10:17:01,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742437_1613 (size=12454) 2024-11-16T10:17:01,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:17:01,191 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/884cfd2256b64e2599138fc85f5b79fa as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/884cfd2256b64e2599138fc85f5b79fa 2024-11-16T10:17:01,193 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 571a63535798b9dbe219ba8ef9231eab/A of 571a63535798b9dbe219ba8ef9231eab into 9ba01a9319ca42a2a07c951bd91692cf(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:17:01,193 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:17:01,193 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., storeName=571a63535798b9dbe219ba8ef9231eab/A, priority=13, startTime=1731752221119; duration=0sec 2024-11-16T10:17:01,193 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:17:01,193 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 571a63535798b9dbe219ba8ef9231eab:A 2024-11-16T10:17:01,195 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ea44957cf2b94c9d8389982a3d8f5446_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ea44957cf2b94c9d8389982a3d8f5446_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:01,195 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 571a63535798b9dbe219ba8ef9231eab/C of 571a63535798b9dbe219ba8ef9231eab into 884cfd2256b64e2599138fc85f5b79fa(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:17:01,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/02e546991e4f4a97a39352ff2fa6b578, store: [table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:17:01,196 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:17:01,196 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., storeName=571a63535798b9dbe219ba8ef9231eab/C, priority=13, startTime=1731752221120; duration=0sec 2024-11-16T10:17:01,196 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:17:01,196 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 571a63535798b9dbe219ba8ef9231eab:C 2024-11-16T10:17:01,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/02e546991e4f4a97a39352ff2fa6b578 is 175, key is test_row_0/A:col10/1731752219869/Put/seqid=0 2024-11-16T10:17:01,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742438_1614 (size=31255) 2024-11-16T10:17:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T10:17:01,601 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/02e546991e4f4a97a39352ff2fa6b578 2024-11-16T10:17:01,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/f105a1beacc148c698f613c8aa83e4f3 is 50, key is test_row_0/B:col10/1731752219869/Put/seqid=0 2024-11-16T10:17:01,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742439_1615 (size=12301) 2024-11-16T10:17:02,010 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/f105a1beacc148c698f613c8aa83e4f3 2024-11-16T10:17:02,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/b0fdfbd027a343df8ea3d545f421af60 is 50, key is test_row_0/C:col10/1731752219869/Put/seqid=0 2024-11-16T10:17:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:02,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:17:02,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742440_1616 (size=12301) 2024-11-16T10:17:02,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752282057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:02,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752282058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:02,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:02,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:02,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:02,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:02,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:02,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:02,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752282164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:02,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752282164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:02,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:02,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752282374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:02,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:02,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752282374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:02,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:02,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:02,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:02,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:02,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T10:17:02,442 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/b0fdfbd027a343df8ea3d545f421af60 2024-11-16T10:17:02,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/02e546991e4f4a97a39352ff2fa6b578 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/02e546991e4f4a97a39352ff2fa6b578 2024-11-16T10:17:02,448 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/02e546991e4f4a97a39352ff2fa6b578, entries=150, sequenceid=329, filesize=30.5 K 2024-11-16T10:17:02,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/f105a1beacc148c698f613c8aa83e4f3 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/f105a1beacc148c698f613c8aa83e4f3 2024-11-16T10:17:02,451 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/f105a1beacc148c698f613c8aa83e4f3, entries=150, sequenceid=329, filesize=12.0 K 2024-11-16T10:17:02,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/b0fdfbd027a343df8ea3d545f421af60 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/b0fdfbd027a343df8ea3d545f421af60 2024-11-16T10:17:02,454 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/b0fdfbd027a343df8ea3d545f421af60, entries=150, sequenceid=329, filesize=12.0 K 2024-11-16T10:17:02,455 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 571a63535798b9dbe219ba8ef9231eab in 1288ms, sequenceid=329, compaction requested=false 2024-11-16T10:17:02,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:17:02,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:02,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-16T10:17:02,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-16T10:17:02,457 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-16T10:17:02,457 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2080 sec 2024-11-16T10:17:02,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.2130 sec 2024-11-16T10:17:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:02,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 571a63535798b9dbe219ba8ef9231eab 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-16T10:17:02,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=A 2024-11-16T10:17:02,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:17:02,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=B 2024-11-16T10:17:02,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:17:02,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=C 2024-11-16T10:17:02,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:17:02,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160ecfe9b09dcd4309ae04a53d50f6db4e_571a63535798b9dbe219ba8ef9231eab is 50, key is test_row_0/A:col10/1731752222688/Put/seqid=0 2024-11-16T10:17:02,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742441_1617 (size=14994) 2024-11-16T10:17:02,717 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:17:02,721 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160ecfe9b09dcd4309ae04a53d50f6db4e_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160ecfe9b09dcd4309ae04a53d50f6db4e_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:02,722 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/e47ac40bc8bd47fc8cccef62df8326d1, store: [table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:17:02,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/e47ac40bc8bd47fc8cccef62df8326d1 is 175, key is test_row_0/A:col10/1731752222688/Put/seqid=0 2024-11-16T10:17:02,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:02,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752282720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:02,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:02,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752282722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:02,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:02,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:02,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:02,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742442_1618 (size=39949) 2024-11-16T10:17:02,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752282834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:02,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:02,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:02,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752282834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:02,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:02,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:02,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:02,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:03,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752283044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:03,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752283044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:03,131 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/e47ac40bc8bd47fc8cccef62df8326d1 2024-11-16T10:17:03,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/65ef99f107cc49e9a82c935e0086fe16 is 50, key is test_row_0/B:col10/1731752222688/Put/seqid=0 2024-11-16T10:17:03,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742443_1619 (size=12301) 2024-11-16T10:17:03,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752283354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:03,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752283354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:03,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:03,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:03,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:03,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/65ef99f107cc49e9a82c935e0086fe16 2024-11-16T10:17:03,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/45c8a93242284069b3d74e7dbf11bf98 is 50, key is test_row_0/C:col10/1731752222688/Put/seqid=0 2024-11-16T10:17:03,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742444_1620 (size=12301) 2024-11-16T10:17:03,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752283864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:03,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752283864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:03,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:03,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:03,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:03,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/45c8a93242284069b3d74e7dbf11bf98 2024-11-16T10:17:03,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/e47ac40bc8bd47fc8cccef62df8326d1 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/e47ac40bc8bd47fc8cccef62df8326d1 2024-11-16T10:17:03,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/e47ac40bc8bd47fc8cccef62df8326d1, entries=200, sequenceid=348, filesize=39.0 K 2024-11-16T10:17:03,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/65ef99f107cc49e9a82c935e0086fe16 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/65ef99f107cc49e9a82c935e0086fe16 2024-11-16T10:17:03,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/65ef99f107cc49e9a82c935e0086fe16, entries=150, sequenceid=348, filesize=12.0 K 2024-11-16T10:17:03,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/45c8a93242284069b3d74e7dbf11bf98 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/45c8a93242284069b3d74e7dbf11bf98 2024-11-16T10:17:03,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/45c8a93242284069b3d74e7dbf11bf98, entries=150, sequenceid=348, filesize=12.0 K 2024-11-16T10:17:03,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 571a63535798b9dbe219ba8ef9231eab in 1296ms, sequenceid=348, compaction requested=true 2024-11-16T10:17:03,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:17:03,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 571a63535798b9dbe219ba8ef9231eab:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:17:03,984 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:17:03,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:17:03,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 571a63535798b9dbe219ba8ef9231eab:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:17:03,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:17:03,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 571a63535798b9dbe219ba8ef9231eab:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:17:03,984 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:17:03,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:17:03,985 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:17:03,986 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 571a63535798b9dbe219ba8ef9231eab/A is initiating minor compaction (all files) 2024-11-16T10:17:03,986 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 571a63535798b9dbe219ba8ef9231eab/A in TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:03,986 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/9ba01a9319ca42a2a07c951bd91692cf, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/02e546991e4f4a97a39352ff2fa6b578, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/e47ac40bc8bd47fc8cccef62df8326d1] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp, totalSize=100.7 K 2024-11-16T10:17:03,986 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:03,986 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. files: [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/9ba01a9319ca42a2a07c951bd91692cf, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/02e546991e4f4a97a39352ff2fa6b578, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/e47ac40bc8bd47fc8cccef62df8326d1] 2024-11-16T10:17:03,986 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:17:03,986 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 571a63535798b9dbe219ba8ef9231eab/B is initiating minor compaction (all files) 2024-11-16T10:17:03,986 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 571a63535798b9dbe219ba8ef9231eab/B in TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:03,987 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/c5891d195a9f48cbbd170b654af86b2a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/f105a1beacc148c698f613c8aa83e4f3, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/65ef99f107cc49e9a82c935e0086fe16] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp, totalSize=36.6 K 2024-11-16T10:17:03,987 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ba01a9319ca42a2a07c951bd91692cf, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731752218690 2024-11-16T10:17:03,987 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting c5891d195a9f48cbbd170b654af86b2a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731752218690 2024-11-16T10:17:03,987 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting f105a1beacc148c698f613c8aa83e4f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731752219869 2024-11-16T10:17:03,987 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02e546991e4f4a97a39352ff2fa6b578, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731752219869 2024-11-16T10:17:03,987 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 65ef99f107cc49e9a82c935e0086fe16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731752222052 2024-11-16T10:17:03,988 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting e47ac40bc8bd47fc8cccef62df8326d1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731752222052 2024-11-16T10:17:04,006 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 571a63535798b9dbe219ba8ef9231eab#B#compaction#522 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:17:04,007 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/35e5155626e74f2d8c5a027e94e6d372 is 50, key is test_row_0/B:col10/1731752222688/Put/seqid=0 2024-11-16T10:17:04,013 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:17:04,021 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116d5d762705a4e4768826defe3235e101d_571a63535798b9dbe219ba8ef9231eab store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:17:04,023 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116d5d762705a4e4768826defe3235e101d_571a63535798b9dbe219ba8ef9231eab, store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:17:04,023 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d5d762705a4e4768826defe3235e101d_571a63535798b9dbe219ba8ef9231eab because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:17:04,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742445_1621 (size=13017) 2024-11-16T10:17:04,036 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/35e5155626e74f2d8c5a027e94e6d372 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/35e5155626e74f2d8c5a027e94e6d372 2024-11-16T10:17:04,046 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 571a63535798b9dbe219ba8ef9231eab/B of 571a63535798b9dbe219ba8ef9231eab into 35e5155626e74f2d8c5a027e94e6d372(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:17:04,046 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:17:04,046 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., storeName=571a63535798b9dbe219ba8ef9231eab/B, priority=13, startTime=1731752223984; duration=0sec 2024-11-16T10:17:04,046 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:17:04,046 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 571a63535798b9dbe219ba8ef9231eab:B 2024-11-16T10:17:04,046 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:17:04,047 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:17:04,048 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 571a63535798b9dbe219ba8ef9231eab/C is initiating minor compaction (all files) 2024-11-16T10:17:04,048 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 571a63535798b9dbe219ba8ef9231eab/C in TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:04,048 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/884cfd2256b64e2599138fc85f5b79fa, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/b0fdfbd027a343df8ea3d545f421af60, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/45c8a93242284069b3d74e7dbf11bf98] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp, totalSize=36.6 K 2024-11-16T10:17:04,049 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 884cfd2256b64e2599138fc85f5b79fa, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731752218690 2024-11-16T10:17:04,050 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting b0fdfbd027a343df8ea3d545f421af60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731752219869 2024-11-16T10:17:04,051 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 45c8a93242284069b3d74e7dbf11bf98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731752222052 2024-11-16T10:17:04,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742446_1622 (size=4469) 2024-11-16T10:17:04,078 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 571a63535798b9dbe219ba8ef9231eab#C#compaction#524 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:17:04,078 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/ddf2f811338a404c8ed9408312ad0607 is 50, key is test_row_0/C:col10/1731752222688/Put/seqid=0 2024-11-16T10:17:04,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742447_1623 (size=13017) 2024-11-16T10:17:04,104 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/ddf2f811338a404c8ed9408312ad0607 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/ddf2f811338a404c8ed9408312ad0607 2024-11-16T10:17:04,115 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 571a63535798b9dbe219ba8ef9231eab/C of 571a63535798b9dbe219ba8ef9231eab into ddf2f811338a404c8ed9408312ad0607(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:17:04,115 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:17:04,115 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., storeName=571a63535798b9dbe219ba8ef9231eab/C, priority=13, startTime=1731752223984; duration=0sec 2024-11-16T10:17:04,115 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:17:04,115 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 571a63535798b9dbe219ba8ef9231eab:C 2024-11-16T10:17:04,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T10:17:04,415 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:17:04,415 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:17:04,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-16T10:17:04,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T10:17:04,417 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:17:04,417 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:17:04,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:17:04,476 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 571a63535798b9dbe219ba8ef9231eab#A#compaction#523 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:17:04,477 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/0391e800f013418982e0901e19689395 is 175, key is test_row_0/A:col10/1731752222688/Put/seqid=0 2024-11-16T10:17:04,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742448_1624 (size=31971) 2024-11-16T10:17:04,484 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/0391e800f013418982e0901e19689395 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/0391e800f013418982e0901e19689395 2024-11-16T10:17:04,488 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 571a63535798b9dbe219ba8ef9231eab/A of 571a63535798b9dbe219ba8ef9231eab into 0391e800f013418982e0901e19689395(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:17:04,488 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:17:04,488 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., storeName=571a63535798b9dbe219ba8ef9231eab/A, priority=13, startTime=1731752223984; duration=0sec 2024-11-16T10:17:04,488 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:17:04,488 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 571a63535798b9dbe219ba8ef9231eab:A 2024-11-16T10:17:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T10:17:04,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T10:17:04,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:04,570 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 571a63535798b9dbe219ba8ef9231eab 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-16T10:17:04,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=A 2024-11-16T10:17:04,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:17:04,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=B 2024-11-16T10:17:04,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:17:04,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=C 2024-11-16T10:17:04,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:17:04,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116697b2940a797457caa404fd717584d43_571a63535798b9dbe219ba8ef9231eab is 50, key is test_row_0/A:col10/1731752222713/Put/seqid=0 2024-11-16T10:17:04,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742449_1625 (size=12454) 2024-11-16T10:17:04,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T10:17:04,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:04,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:17:04,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:04,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752284907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:04,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:04,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752284910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:04,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:04,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:04,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:04,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:17:04,992 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116697b2940a797457caa404fd717584d43_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116697b2940a797457caa404fd717584d43_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:04,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/5ac85e2682f543a7bc6aebf3491c5dee, store: [table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:17:04,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/5ac85e2682f543a7bc6aebf3491c5dee is 175, key is test_row_0/A:col10/1731752222713/Put/seqid=0 2024-11-16T10:17:04,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742450_1626 (size=31255) 2024-11-16T10:17:05,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:05,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752285014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:05,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:05,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752285024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:05,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T10:17:05,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:05,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43504 deadline: 1731752285134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:05,136 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,45417,1731752048912 for 'test_row_0' in 571a63535798b9dbe219ba8ef9231eab of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:17:05,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:05,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:05,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:05,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:05,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752285224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:05,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:05,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:05,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:05,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752285234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:05,397 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=366, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/5ac85e2682f543a7bc6aebf3491c5dee 2024-11-16T10:17:05,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/afecacc4a2404bf08fd0896a072df2b6 is 50, key is test_row_0/B:col10/1731752222713/Put/seqid=0 2024-11-16T10:17:05,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742451_1627 (size=12301) 2024-11-16T10:17:05,409 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/afecacc4a2404bf08fd0896a072df2b6 2024-11-16T10:17:05,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/54de8ca623e04d488e804754217a8e9e is 50, key is test_row_0/C:col10/1731752222713/Put/seqid=0 2024-11-16T10:17:05,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742452_1628 (size=12301) 2024-11-16T10:17:05,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752285534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:05,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:05,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:05,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:05,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:05,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752285543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:05,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:05,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:05,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:05,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T10:17:05,818 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/54de8ca623e04d488e804754217a8e9e 2024-11-16T10:17:05,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/5ac85e2682f543a7bc6aebf3491c5dee as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/5ac85e2682f543a7bc6aebf3491c5dee 2024-11-16T10:17:05,825 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/5ac85e2682f543a7bc6aebf3491c5dee, entries=150, sequenceid=366, filesize=30.5 K 2024-11-16T10:17:05,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/afecacc4a2404bf08fd0896a072df2b6 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/afecacc4a2404bf08fd0896a072df2b6 2024-11-16T10:17:05,828 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/afecacc4a2404bf08fd0896a072df2b6, entries=150, sequenceid=366, filesize=12.0 K 2024-11-16T10:17:05,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/54de8ca623e04d488e804754217a8e9e as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/54de8ca623e04d488e804754217a8e9e 2024-11-16T10:17:05,831 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/54de8ca623e04d488e804754217a8e9e, entries=150, sequenceid=366, filesize=12.0 K 2024-11-16T10:17:05,832 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 571a63535798b9dbe219ba8ef9231eab in 1262ms, sequenceid=366, compaction requested=false 2024-11-16T10:17:05,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:17:05,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:05,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-16T10:17:05,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-16T10:17:05,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-16T10:17:05,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4160 sec 2024-11-16T10:17:05,837 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.4210 sec 2024-11-16T10:17:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:06,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 571a63535798b9dbe219ba8ef9231eab 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-16T10:17:06,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=A 2024-11-16T10:17:06,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:17:06,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=B 2024-11-16T10:17:06,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:17:06,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=C 2024-11-16T10:17:06,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:17:06,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c0a8ada2fde7473c8538685d1cc35782_571a63535798b9dbe219ba8ef9231eab is 50, key is test_row_0/A:col10/1731752224904/Put/seqid=0 2024-11-16T10:17:06,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742453_1629 (size=12454) 2024-11-16T10:17:06,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752286072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:06,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:06,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:06,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:06,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752286075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:06,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:06,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:06,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:06,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752286184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:06,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752286184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:06,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752286394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:06,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:06,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:06,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752286394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:06,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:06,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:06,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:06,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:06,454 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:17:06,457 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c0a8ada2fde7473c8538685d1cc35782_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c0a8ada2fde7473c8538685d1cc35782_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:06,458 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/9d116a973fa1401a928b5bbd6b683abc, store: [table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:17:06,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/9d116a973fa1401a928b5bbd6b683abc is 175, key is test_row_0/A:col10/1731752224904/Put/seqid=0 2024-11-16T10:17:06,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742454_1630 (size=31255) 2024-11-16T10:17:06,463 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=384, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/9d116a973fa1401a928b5bbd6b683abc 2024-11-16T10:17:06,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/dcd865fcdcaf41b696980e4e3550fb9c is 50, key is test_row_0/B:col10/1731752224904/Put/seqid=0 2024-11-16T10:17:06,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742455_1631 (size=12301) 2024-11-16T10:17:06,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T10:17:06,564 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:17:06,564 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:17:06,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-16T10:17:06,565 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:17:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T10:17:06,566 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:17:06,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:17:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T10:17:06,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752286704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:06,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:06,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:06,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:06,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752286704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:06,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:06,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:06,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:06,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T10:17:06,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:06,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:17:06,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:06,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:06,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:06,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T10:17:06,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:06,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:17:06,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:06,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:06,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:06,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:06,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/dcd865fcdcaf41b696980e4e3550fb9c 2024-11-16T10:17:06,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/aa42c993cad74408b4a33a793cd34824 is 50, key is test_row_0/C:col10/1731752224904/Put/seqid=0 2024-11-16T10:17:06,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742456_1632 (size=12301) 2024-11-16T10:17:06,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T10:17:06,909 INFO [Thread-2403 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2403. 2024-11-16T10:17:06,909 DEBUG [Thread-2403 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:17:06,909 DEBUG [Thread-2403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:06,909 DEBUG [Thread-2403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:06,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:17:06,911 INFO [Thread-2409 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2409. 2024-11-16T10:17:06,911 INFO [Thread-2407 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2407. 2024-11-16T10:17:06,911 DEBUG [Thread-2409 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:17:06,911 DEBUG [Thread-2407 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:17:06,911 DEBUG [Thread-2409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:06,911 DEBUG [Thread-2407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:06,911 DEBUG [Thread-2409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:06,911 DEBUG [Thread-2407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:06,911 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:17:06,911 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:17:06,914 INFO [Thread-2405 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2405. 2024-11-16T10:17:06,914 DEBUG [Thread-2405 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:17:06,914 DEBUG [Thread-2405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:06,914 DEBUG [Thread-2405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:06,914 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:17:06,915 INFO [Thread-2401 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2401. 2024-11-16T10:17:06,915 DEBUG [Thread-2401 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:17:06,915 DEBUG [Thread-2401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:06,915 DEBUG [Thread-2401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:06,915 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:17:07,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T10:17:07,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:07,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:17:07,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:07,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:07,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:07,028 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T10:17:07,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T10:17:07,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:07,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:17:07,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:07,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:07,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:17:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T10:17:07,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:07,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:17:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1731752287215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45417 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43456 deadline: 1731752287215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:17:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., hostname=b16ff38458c1,45417,1731752048912, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:17:07,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/aa42c993cad74408b4a33a793cd34824 2024-11-16T10:17:07,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/9d116a973fa1401a928b5bbd6b683abc as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/9d116a973fa1401a928b5bbd6b683abc 2024-11-16T10:17:07,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/9d116a973fa1401a928b5bbd6b683abc, entries=150, sequenceid=384, filesize=30.5 K 2024-11-16T10:17:07,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/dcd865fcdcaf41b696980e4e3550fb9c as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/dcd865fcdcaf41b696980e4e3550fb9c 2024-11-16T10:17:07,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/dcd865fcdcaf41b696980e4e3550fb9c, entries=150, sequenceid=384, filesize=12.0 K 2024-11-16T10:17:07,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/aa42c993cad74408b4a33a793cd34824 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/aa42c993cad74408b4a33a793cd34824 2024-11-16T10:17:07,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/aa42c993cad74408b4a33a793cd34824, entries=150, sequenceid=384, filesize=12.0 K 2024-11-16T10:17:07,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 571a63535798b9dbe219ba8ef9231eab in 1261ms, sequenceid=384, compaction requested=true 2024-11-16T10:17:07,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:17:07,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 571a63535798b9dbe219ba8ef9231eab:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:17:07,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:17:07,307 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:17:07,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 571a63535798b9dbe219ba8ef9231eab:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:17:07,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:17:07,307 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:17:07,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 571a63535798b9dbe219ba8ef9231eab:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:17:07,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:17:07,308 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:17:07,308 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:17:07,308 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 571a63535798b9dbe219ba8ef9231eab/B is initiating minor compaction (all files) 2024-11-16T10:17:07,308 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1541): 571a63535798b9dbe219ba8ef9231eab/A is initiating minor compaction (all files) 2024-11-16T10:17:07,308 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 571a63535798b9dbe219ba8ef9231eab/B in TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:07,308 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 571a63535798b9dbe219ba8ef9231eab/A in TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:07,308 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/0391e800f013418982e0901e19689395, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/5ac85e2682f543a7bc6aebf3491c5dee, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/9d116a973fa1401a928b5bbd6b683abc] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp, totalSize=92.3 K 2024-11-16T10:17:07,308 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/35e5155626e74f2d8c5a027e94e6d372, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/afecacc4a2404bf08fd0896a072df2b6, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/dcd865fcdcaf41b696980e4e3550fb9c] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp, totalSize=36.7 K 2024-11-16T10:17:07,308 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:07,308 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. files: [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/0391e800f013418982e0901e19689395, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/5ac85e2682f543a7bc6aebf3491c5dee, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/9d116a973fa1401a928b5bbd6b683abc] 2024-11-16T10:17:07,308 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 35e5155626e74f2d8c5a027e94e6d372, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731752222052 2024-11-16T10:17:07,308 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0391e800f013418982e0901e19689395, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731752222052 2024-11-16T10:17:07,308 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting afecacc4a2404bf08fd0896a072df2b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731752222707 2024-11-16T10:17:07,308 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ac85e2682f543a7bc6aebf3491c5dee, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731752222707 2024-11-16T10:17:07,308 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting dcd865fcdcaf41b696980e4e3550fb9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731752224904 2024-11-16T10:17:07,308 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d116a973fa1401a928b5bbd6b683abc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731752224904 2024-11-16T10:17:07,312 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:17:07,313 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411166cedbf2b501e444aa88064f069dd9370_571a63535798b9dbe219ba8ef9231eab store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:17:07,313 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 571a63535798b9dbe219ba8ef9231eab#B#compaction#531 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:17:07,313 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/9caf2c479bcf4b1fb2e6374def677678 is 50, key is test_row_0/B:col10/1731752224904/Put/seqid=0 2024-11-16T10:17:07,315 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411166cedbf2b501e444aa88064f069dd9370_571a63535798b9dbe219ba8ef9231eab, store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:17:07,315 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166cedbf2b501e444aa88064f069dd9370_571a63535798b9dbe219ba8ef9231eab because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:17:07,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742457_1633 (size=13119) 2024-11-16T10:17:07,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742458_1634 (size=4469) 2024-11-16T10:17:07,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T10:17:07,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:07,332 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 571a63535798b9dbe219ba8ef9231eab 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T10:17:07,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=A 2024-11-16T10:17:07,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:17:07,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=B 2024-11-16T10:17:07,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:17:07,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=C 2024-11-16T10:17:07,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:17:07,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ee46f80885884ab7b765326a0d30956c_571a63535798b9dbe219ba8ef9231eab is 50, key is test_row_0/A:col10/1731752226059/Put/seqid=0 2024-11-16T10:17:07,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742459_1635 (size=12454) 2024-11-16T10:17:07,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T10:17:07,720 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 571a63535798b9dbe219ba8ef9231eab#A#compaction#532 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:17:07,721 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/3ee9d241fc7e443e99f4d3c8232f7106 is 175, key is test_row_0/A:col10/1731752224904/Put/seqid=0 2024-11-16T10:17:07,727 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/9caf2c479bcf4b1fb2e6374def677678 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/9caf2c479bcf4b1fb2e6374def677678 2024-11-16T10:17:07,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742460_1636 (size=32073) 2024-11-16T10:17:07,731 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 571a63535798b9dbe219ba8ef9231eab/B of 571a63535798b9dbe219ba8ef9231eab into 9caf2c479bcf4b1fb2e6374def677678(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:17:07,732 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:17:07,732 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., storeName=571a63535798b9dbe219ba8ef9231eab/B, priority=13, startTime=1731752227307; duration=0sec 2024-11-16T10:17:07,732 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:17:07,732 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 571a63535798b9dbe219ba8ef9231eab:B 2024-11-16T10:17:07,732 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:17:07,733 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:17:07,733 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1541): 571a63535798b9dbe219ba8ef9231eab/C is initiating minor compaction (all files) 2024-11-16T10:17:07,733 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 571a63535798b9dbe219ba8ef9231eab/C in TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:07,733 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/ddf2f811338a404c8ed9408312ad0607, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/54de8ca623e04d488e804754217a8e9e, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/aa42c993cad74408b4a33a793cd34824] into tmpdir=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp, totalSize=36.7 K 2024-11-16T10:17:07,733 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting ddf2f811338a404c8ed9408312ad0607, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731752222052 2024-11-16T10:17:07,734 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting 54de8ca623e04d488e804754217a8e9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731752222707 2024-11-16T10:17:07,734 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] compactions.Compactor(225): Compacting aa42c993cad74408b4a33a793cd34824, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731752224904 2024-11-16T10:17:07,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:17:07,743 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 571a63535798b9dbe219ba8ef9231eab#C#compaction#534 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:17:07,744 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/d86f7734025642be8294b421115395bd is 50, key is test_row_0/C:col10/1731752224904/Put/seqid=0 2024-11-16T10:17:07,744 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ee46f80885884ab7b765326a0d30956c_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ee46f80885884ab7b765326a0d30956c_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:07,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/4a5eeb08c8044140b557690188253522, store: [table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:17:07,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/4a5eeb08c8044140b557690188253522 is 175, key is test_row_0/A:col10/1731752226059/Put/seqid=0 2024-11-16T10:17:07,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742461_1637 (size=13119) 2024-11-16T10:17:07,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742462_1638 (size=31255) 2024-11-16T10:17:07,749 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=398, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/4a5eeb08c8044140b557690188253522 2024-11-16T10:17:07,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/ac55138ba4914934b1c906c5fe8cab38 is 50, key is test_row_0/B:col10/1731752226059/Put/seqid=0 2024-11-16T10:17:07,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742463_1639 (size=12301) 2024-11-16T10:17:08,136 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/3ee9d241fc7e443e99f4d3c8232f7106 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/3ee9d241fc7e443e99f4d3c8232f7106 2024-11-16T10:17:08,142 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 571a63535798b9dbe219ba8ef9231eab/A of 571a63535798b9dbe219ba8ef9231eab into 3ee9d241fc7e443e99f4d3c8232f7106(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:17:08,142 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:17:08,142 INFO [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., storeName=571a63535798b9dbe219ba8ef9231eab/A, priority=13, startTime=1731752227307; duration=0sec 2024-11-16T10:17:08,142 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:17:08,142 DEBUG [RS:0;b16ff38458c1:45417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 571a63535798b9dbe219ba8ef9231eab:A 2024-11-16T10:17:08,153 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/d86f7734025642be8294b421115395bd as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/d86f7734025642be8294b421115395bd 2024-11-16T10:17:08,156 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/ac55138ba4914934b1c906c5fe8cab38 2024-11-16T10:17:08,158 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 571a63535798b9dbe219ba8ef9231eab/C of 571a63535798b9dbe219ba8ef9231eab into d86f7734025642be8294b421115395bd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:17:08,158 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:17:08,158 INFO [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., storeName=571a63535798b9dbe219ba8ef9231eab/C, priority=13, startTime=1731752227307; duration=0sec 2024-11-16T10:17:08,158 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:17:08,158 DEBUG [RS:0;b16ff38458c1:45417-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 571a63535798b9dbe219ba8ef9231eab:C 2024-11-16T10:17:08,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/ea98bfb4c4614abd8b45b3dca6f6200a is 50, key is test_row_0/C:col10/1731752226059/Put/seqid=0 2024-11-16T10:17:08,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742464_1640 (size=12301) 2024-11-16T10:17:08,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45417 {}] regionserver.HRegion(8855): Flush requested on 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:08,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. as already flushing 2024-11-16T10:17:08,225 INFO [Thread-2396 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2396. 2024-11-16T10:17:08,225 INFO [Thread-2390 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2390. 2024-11-16T10:17:08,225 DEBUG [Thread-2390 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:17:08,225 DEBUG [Thread-2396 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:17:08,225 DEBUG [Thread-2390 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:08,225 DEBUG [Thread-2396 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:08,225 DEBUG [Thread-2390 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:08,225 DEBUG [Thread-2396 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:08,225 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:17:08,225 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:17:08,568 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/ea98bfb4c4614abd8b45b3dca6f6200a 2024-11-16T10:17:08,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/4a5eeb08c8044140b557690188253522 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/4a5eeb08c8044140b557690188253522 2024-11-16T10:17:08,574 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/4a5eeb08c8044140b557690188253522, entries=150, sequenceid=398, filesize=30.5 K 2024-11-16T10:17:08,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/ac55138ba4914934b1c906c5fe8cab38 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/ac55138ba4914934b1c906c5fe8cab38 2024-11-16T10:17:08,578 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/ac55138ba4914934b1c906c5fe8cab38, entries=150, sequenceid=398, filesize=12.0 K 2024-11-16T10:17:08,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/ea98bfb4c4614abd8b45b3dca6f6200a as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/ea98bfb4c4614abd8b45b3dca6f6200a 2024-11-16T10:17:08,582 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/ea98bfb4c4614abd8b45b3dca6f6200a, entries=150, sequenceid=398, filesize=12.0 K 2024-11-16T10:17:08,582 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=13.42 KB/13740 for 571a63535798b9dbe219ba8ef9231eab in 1250ms, sequenceid=398, compaction requested=false 2024-11-16T10:17:08,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 571a63535798b9dbe219ba8ef9231eab: 2024-11-16T10:17:08,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:08,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-16T10:17:08,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-16T10:17:08,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-16T10:17:08,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0180 sec 2024-11-16T10:17:08,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.0220 sec 2024-11-16T10:17:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T10:17:08,715 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:17:10,966 INFO [Thread-2392 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2392. 2024-11-16T10:17:10,967 DEBUG [Thread-2392 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:17:10,967 DEBUG [Thread-2392 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:10,967 DEBUG [Thread-2392 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:10,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:17:11,007 INFO [Thread-2398 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2398. 2024-11-16T10:17:11,007 DEBUG [Thread-2398 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:17:11,007 DEBUG [Thread-2398 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:11,008 DEBUG [Thread-2398 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:11,008 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:17:15,239 INFO [Thread-2394 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2394. 2024-11-16T10:17:15,239 DEBUG [Thread-2394 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:17:15,240 DEBUG [Thread-2394 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:15,240 DEBUG [Thread-2394 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:15,240 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:17:15,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T10:17:15,241 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 121 2024-11-16T10:17:15,241 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-16T10:17:15,241 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 3 2024-11-16T10:17:15,241 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 115 2024-11-16T10:17:15,241 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-16T10:17:15,241 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T10:17:15,241 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2658 2024-11-16T10:17:15,241 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2767 2024-11-16T10:17:15,241 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2664 2024-11-16T10:17:15,241 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2677 2024-11-16T10:17:15,241 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2758 2024-11-16T10:17:15,241 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T10:17:15,241 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T10:17:15,242 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T10:17:15,242 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:15,242 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:15,242 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:17:15,244 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T10:17:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T10:17:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T10:17:15,248 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752235248"}]},"ts":"1731752235248"} 2024-11-16T10:17:15,250 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T10:17:15,250 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T10:17:15,251 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T10:17:15,253 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=571a63535798b9dbe219ba8ef9231eab, UNASSIGN}] 2024-11-16T10:17:15,253 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=571a63535798b9dbe219ba8ef9231eab, UNASSIGN 2024-11-16T10:17:15,254 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=571a63535798b9dbe219ba8ef9231eab, regionState=CLOSING, regionLocation=b16ff38458c1,45417,1731752048912 2024-11-16T10:17:15,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=571a63535798b9dbe219ba8ef9231eab, UNASSIGN because future has completed 2024-11-16T10:17:15,256 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T10:17:15,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912}] 2024-11-16T10:17:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T10:17:15,408 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(122): Close 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:15,408 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T10:17:15,408 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1722): Closing 571a63535798b9dbe219ba8ef9231eab, disabling compactions & flushes 2024-11-16T10:17:15,408 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:15,408 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:15,408 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. after waiting 0 ms 2024-11-16T10:17:15,408 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:15,408 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(2902): Flushing 571a63535798b9dbe219ba8ef9231eab 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-16T10:17:15,408 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=A 2024-11-16T10:17:15,408 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:17:15,409 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=B 2024-11-16T10:17:15,409 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:17:15,409 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 571a63535798b9dbe219ba8ef9231eab, store=C 2024-11-16T10:17:15,409 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T10:17:15,416 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116625c5ce471e942358a2442cc570c11bd_571a63535798b9dbe219ba8ef9231eab is 50, key is test_row_0/A:col10/1731752235236/Put/seqid=0 2024-11-16T10:17:15,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742465_1641 (size=9914) 2024-11-16T10:17:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T10:17:15,822 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:17:15,829 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116625c5ce471e942358a2442cc570c11bd_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116625c5ce471e942358a2442cc570c11bd_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:15,831 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/57abf78d79604c299daed3ff1c9fb71e, store: [table=TestAcidGuarantees family=A region=571a63535798b9dbe219ba8ef9231eab] 2024-11-16T10:17:15,832 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/57abf78d79604c299daed3ff1c9fb71e is 175, key is test_row_0/A:col10/1731752235236/Put/seqid=0 2024-11-16T10:17:15,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742466_1642 (size=22561) 2024-11-16T10:17:15,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T10:17:16,237 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=409, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/57abf78d79604c299daed3ff1c9fb71e 2024-11-16T10:17:16,242 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/563b71447431461c892f0d321090fb86 is 50, key is test_row_0/B:col10/1731752235236/Put/seqid=0 2024-11-16T10:17:16,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742467_1643 (size=9857) 2024-11-16T10:17:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T10:17:16,645 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/563b71447431461c892f0d321090fb86 2024-11-16T10:17:16,658 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/05d6ea930cf7417bb59794c36829cdea is 50, key is test_row_0/C:col10/1731752235236/Put/seqid=0 2024-11-16T10:17:16,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742468_1644 (size=9857) 2024-11-16T10:17:17,062 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/05d6ea930cf7417bb59794c36829cdea 2024-11-16T10:17:17,066 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/A/57abf78d79604c299daed3ff1c9fb71e as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/57abf78d79604c299daed3ff1c9fb71e 2024-11-16T10:17:17,069 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/57abf78d79604c299daed3ff1c9fb71e, entries=100, sequenceid=409, filesize=22.0 K 2024-11-16T10:17:17,070 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/B/563b71447431461c892f0d321090fb86 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/563b71447431461c892f0d321090fb86 2024-11-16T10:17:17,072 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/563b71447431461c892f0d321090fb86, entries=100, sequenceid=409, filesize=9.6 K 2024-11-16T10:17:17,073 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/.tmp/C/05d6ea930cf7417bb59794c36829cdea as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/05d6ea930cf7417bb59794c36829cdea 2024-11-16T10:17:17,075 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/05d6ea930cf7417bb59794c36829cdea, entries=100, sequenceid=409, filesize=9.6 K 2024-11-16T10:17:17,075 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 571a63535798b9dbe219ba8ef9231eab in 1667ms, sequenceid=409, compaction requested=true 2024-11-16T10:17:17,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/8c9d3d7d22ff40d0873213dd67b8bfd2, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/baee30d5060b41599c28798bfb448211, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/e0d2b236ae304176bc3f6928d18af6af, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/1498a57065a1483bb5144cdf4d2db430, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/3589724e517d4d818f5447d5e8fcf6b5, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/317d9beb089e42c9a6f796d43a192249, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/c4849065beee4411a9307bbb3d3a2be5, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/74f50b8912a24a0598801681ad0d1a6b, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/cca0b2726f8445008e7c205bdcb9862f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/787f57b28f3842f387fd1fe79c3d2995, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/d6741a1ec9bf4a0b8936ec4aaad447fc, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/100e99f14a554fd98fc7a920044d73f8, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/74b601071c96401791cd6fa7a2ced536, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/d97fc00ad2df4bf0966cbfa6b6b9b267, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/418031ada2e448ae812761f04da32ae2, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/a005469487354ec59ed800778d6e4219, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/5e3ab2186f624faaadf5cfeb75c5700f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/93a8dbdcf3dd44cba1fc16dd5957ae0d, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/9ba01a9319ca42a2a07c951bd91692cf, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/02e546991e4f4a97a39352ff2fa6b578, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/e47ac40bc8bd47fc8cccef62df8326d1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/0391e800f013418982e0901e19689395, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/5ac85e2682f543a7bc6aebf3491c5dee, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/9d116a973fa1401a928b5bbd6b683abc] to archive 2024-11-16T10:17:17,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:17:17,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/8c9d3d7d22ff40d0873213dd67b8bfd2 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/8c9d3d7d22ff40d0873213dd67b8bfd2 2024-11-16T10:17:17,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/baee30d5060b41599c28798bfb448211 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/baee30d5060b41599c28798bfb448211 2024-11-16T10:17:17,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/e0d2b236ae304176bc3f6928d18af6af to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/e0d2b236ae304176bc3f6928d18af6af 2024-11-16T10:17:17,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/1498a57065a1483bb5144cdf4d2db430 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/1498a57065a1483bb5144cdf4d2db430 2024-11-16T10:17:17,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/3589724e517d4d818f5447d5e8fcf6b5 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/3589724e517d4d818f5447d5e8fcf6b5 2024-11-16T10:17:17,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/317d9beb089e42c9a6f796d43a192249 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/317d9beb089e42c9a6f796d43a192249 2024-11-16T10:17:17,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/c4849065beee4411a9307bbb3d3a2be5 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/c4849065beee4411a9307bbb3d3a2be5 2024-11-16T10:17:17,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/74f50b8912a24a0598801681ad0d1a6b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/74f50b8912a24a0598801681ad0d1a6b 2024-11-16T10:17:17,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/cca0b2726f8445008e7c205bdcb9862f to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/cca0b2726f8445008e7c205bdcb9862f 2024-11-16T10:17:17,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/787f57b28f3842f387fd1fe79c3d2995 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/787f57b28f3842f387fd1fe79c3d2995 2024-11-16T10:17:17,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/d6741a1ec9bf4a0b8936ec4aaad447fc to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/d6741a1ec9bf4a0b8936ec4aaad447fc 2024-11-16T10:17:17,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/100e99f14a554fd98fc7a920044d73f8 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/100e99f14a554fd98fc7a920044d73f8 2024-11-16T10:17:17,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/74b601071c96401791cd6fa7a2ced536 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/74b601071c96401791cd6fa7a2ced536 2024-11-16T10:17:17,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/d97fc00ad2df4bf0966cbfa6b6b9b267 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/d97fc00ad2df4bf0966cbfa6b6b9b267 2024-11-16T10:17:17,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/418031ada2e448ae812761f04da32ae2 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/418031ada2e448ae812761f04da32ae2 2024-11-16T10:17:17,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/a005469487354ec59ed800778d6e4219 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/a005469487354ec59ed800778d6e4219 2024-11-16T10:17:17,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/5e3ab2186f624faaadf5cfeb75c5700f to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/5e3ab2186f624faaadf5cfeb75c5700f 2024-11-16T10:17:17,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/93a8dbdcf3dd44cba1fc16dd5957ae0d to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/93a8dbdcf3dd44cba1fc16dd5957ae0d 2024-11-16T10:17:17,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/9ba01a9319ca42a2a07c951bd91692cf to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/9ba01a9319ca42a2a07c951bd91692cf 2024-11-16T10:17:17,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/02e546991e4f4a97a39352ff2fa6b578 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/02e546991e4f4a97a39352ff2fa6b578 2024-11-16T10:17:17,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/e47ac40bc8bd47fc8cccef62df8326d1 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/e47ac40bc8bd47fc8cccef62df8326d1 2024-11-16T10:17:17,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/0391e800f013418982e0901e19689395 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/0391e800f013418982e0901e19689395 2024-11-16T10:17:17,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/5ac85e2682f543a7bc6aebf3491c5dee to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/5ac85e2682f543a7bc6aebf3491c5dee 2024-11-16T10:17:17,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/9d116a973fa1401a928b5bbd6b683abc to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/9d116a973fa1401a928b5bbd6b683abc 2024-11-16T10:17:17,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/d9b5a98409fb4e8b88a431f32e8384be, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/54d456119f954dc6961f74b8ff6f9427, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/323693d04fe54ba8a4e05ecf8f31d40d, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/8a5e689a6a4042f996b57b4cf6328c37, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/1c5df67f322646ffa719cfdd5a0990e4, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/8ce58593de324ebfa40a0ccd01c1204b, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/9ca8869e43e24c0e80d6cb18a218d467, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/fe509def5cc74795a7e08e44058440ba, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/7f2748e818434a8e81307bc972a8230f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/1be679472b8749a29cf84676c0e794c9, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/c1ae31a137744cb98e8d43884cacad84, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/7161b6ea279c42fd8c8704d98c0ba1d8, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/ff753a007f914ac79d273a8dcdfbf08c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/979a34ca31a44012b291dd9846658e50, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/24770de74b6e417a90b5dbc7c503c4b1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/823d9995fb1c4325a9461f2c7d9a8037, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/e53da9d873324413bf3f1d29fdc31154, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/c5891d195a9f48cbbd170b654af86b2a, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/5497c416c16b466e8814bf4e5d2470a1, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/f105a1beacc148c698f613c8aa83e4f3, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/35e5155626e74f2d8c5a027e94e6d372, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/65ef99f107cc49e9a82c935e0086fe16, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/afecacc4a2404bf08fd0896a072df2b6, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/dcd865fcdcaf41b696980e4e3550fb9c] to archive 2024-11-16T10:17:17,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:17:17,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/d9b5a98409fb4e8b88a431f32e8384be to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/d9b5a98409fb4e8b88a431f32e8384be 2024-11-16T10:17:17,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/54d456119f954dc6961f74b8ff6f9427 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/54d456119f954dc6961f74b8ff6f9427 2024-11-16T10:17:17,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/323693d04fe54ba8a4e05ecf8f31d40d to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/323693d04fe54ba8a4e05ecf8f31d40d 2024-11-16T10:17:17,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/8a5e689a6a4042f996b57b4cf6328c37 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/8a5e689a6a4042f996b57b4cf6328c37 2024-11-16T10:17:17,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/1c5df67f322646ffa719cfdd5a0990e4 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/1c5df67f322646ffa719cfdd5a0990e4 2024-11-16T10:17:17,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/8ce58593de324ebfa40a0ccd01c1204b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/8ce58593de324ebfa40a0ccd01c1204b 2024-11-16T10:17:17,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/9ca8869e43e24c0e80d6cb18a218d467 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/9ca8869e43e24c0e80d6cb18a218d467 2024-11-16T10:17:17,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/fe509def5cc74795a7e08e44058440ba to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/fe509def5cc74795a7e08e44058440ba 2024-11-16T10:17:17,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/7f2748e818434a8e81307bc972a8230f to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/7f2748e818434a8e81307bc972a8230f 2024-11-16T10:17:17,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/1be679472b8749a29cf84676c0e794c9 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/1be679472b8749a29cf84676c0e794c9 2024-11-16T10:17:17,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/c1ae31a137744cb98e8d43884cacad84 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/c1ae31a137744cb98e8d43884cacad84 2024-11-16T10:17:17,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/7161b6ea279c42fd8c8704d98c0ba1d8 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/7161b6ea279c42fd8c8704d98c0ba1d8 2024-11-16T10:17:17,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/ff753a007f914ac79d273a8dcdfbf08c to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/ff753a007f914ac79d273a8dcdfbf08c 2024-11-16T10:17:17,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/979a34ca31a44012b291dd9846658e50 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/979a34ca31a44012b291dd9846658e50 2024-11-16T10:17:17,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/24770de74b6e417a90b5dbc7c503c4b1 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/24770de74b6e417a90b5dbc7c503c4b1 2024-11-16T10:17:17,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/823d9995fb1c4325a9461f2c7d9a8037 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/823d9995fb1c4325a9461f2c7d9a8037 2024-11-16T10:17:17,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/e53da9d873324413bf3f1d29fdc31154 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/e53da9d873324413bf3f1d29fdc31154 2024-11-16T10:17:17,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/c5891d195a9f48cbbd170b654af86b2a to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/c5891d195a9f48cbbd170b654af86b2a 2024-11-16T10:17:17,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/5497c416c16b466e8814bf4e5d2470a1 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/5497c416c16b466e8814bf4e5d2470a1 2024-11-16T10:17:17,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/f105a1beacc148c698f613c8aa83e4f3 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/f105a1beacc148c698f613c8aa83e4f3 2024-11-16T10:17:17,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/35e5155626e74f2d8c5a027e94e6d372 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/35e5155626e74f2d8c5a027e94e6d372 2024-11-16T10:17:17,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/65ef99f107cc49e9a82c935e0086fe16 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/65ef99f107cc49e9a82c935e0086fe16 2024-11-16T10:17:17,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/afecacc4a2404bf08fd0896a072df2b6 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/afecacc4a2404bf08fd0896a072df2b6 2024-11-16T10:17:17,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/dcd865fcdcaf41b696980e4e3550fb9c to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/dcd865fcdcaf41b696980e4e3550fb9c 2024-11-16T10:17:17,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/51e2a975fc964a8885cf83af67693b4b, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/b6bbc15cad8c48fabb5a6f1afa62f619, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/42f45e8b70f24585bf631e57cd30be95, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/61d24a7ba099460aa2da37464e22aa31, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3d32caf57af441c3bc2bac37c990e2a5, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3b1396c3a2384b1f8068e4d2eeda7c92, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/53aa52d5d5574d0ca1f3d34d734740fb, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/71b4766951d64ccabab7ffe61f8c5379, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3a543f85da214aceb97e03f21f82b0a3, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/19539025e7b646ba9fd928d2c504864c, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3862efda909e4d868dc52c4095d99822, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/65c0aae7a4624513a1177cc07c252f43, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/0160311842f74c7099f49caaa4b57c7b, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/4cca84c8ec274381a6c222d9bcfdc470, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/fb9fcbb0629a4e1994bb274ccf64aa8f, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/be684bf28eef4ca18071ea347cfd61f9, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/2a2a2bfd102844368a6733c4fc5eb104, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/884cfd2256b64e2599138fc85f5b79fa, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/e2e97daeaa1d4babb424b4b756f8bd36, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/b0fdfbd027a343df8ea3d545f421af60, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/ddf2f811338a404c8ed9408312ad0607, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/45c8a93242284069b3d74e7dbf11bf98, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/54de8ca623e04d488e804754217a8e9e, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/aa42c993cad74408b4a33a793cd34824] to archive 2024-11-16T10:17:17,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:17:17,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/51e2a975fc964a8885cf83af67693b4b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/51e2a975fc964a8885cf83af67693b4b 2024-11-16T10:17:17,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/b6bbc15cad8c48fabb5a6f1afa62f619 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/b6bbc15cad8c48fabb5a6f1afa62f619 2024-11-16T10:17:17,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/42f45e8b70f24585bf631e57cd30be95 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/42f45e8b70f24585bf631e57cd30be95 2024-11-16T10:17:17,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/61d24a7ba099460aa2da37464e22aa31 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/61d24a7ba099460aa2da37464e22aa31 2024-11-16T10:17:17,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3d32caf57af441c3bc2bac37c990e2a5 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3d32caf57af441c3bc2bac37c990e2a5 2024-11-16T10:17:17,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3b1396c3a2384b1f8068e4d2eeda7c92 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3b1396c3a2384b1f8068e4d2eeda7c92 2024-11-16T10:17:17,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/53aa52d5d5574d0ca1f3d34d734740fb to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/53aa52d5d5574d0ca1f3d34d734740fb 2024-11-16T10:17:17,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/71b4766951d64ccabab7ffe61f8c5379 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/71b4766951d64ccabab7ffe61f8c5379 2024-11-16T10:17:17,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3a543f85da214aceb97e03f21f82b0a3 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3a543f85da214aceb97e03f21f82b0a3 2024-11-16T10:17:17,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/19539025e7b646ba9fd928d2c504864c to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/19539025e7b646ba9fd928d2c504864c 2024-11-16T10:17:17,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3862efda909e4d868dc52c4095d99822 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/3862efda909e4d868dc52c4095d99822 2024-11-16T10:17:17,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/65c0aae7a4624513a1177cc07c252f43 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/65c0aae7a4624513a1177cc07c252f43 2024-11-16T10:17:17,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/0160311842f74c7099f49caaa4b57c7b to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/0160311842f74c7099f49caaa4b57c7b 2024-11-16T10:17:17,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/4cca84c8ec274381a6c222d9bcfdc470 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/4cca84c8ec274381a6c222d9bcfdc470 2024-11-16T10:17:17,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/fb9fcbb0629a4e1994bb274ccf64aa8f to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/fb9fcbb0629a4e1994bb274ccf64aa8f 2024-11-16T10:17:17,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/be684bf28eef4ca18071ea347cfd61f9 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/be684bf28eef4ca18071ea347cfd61f9 2024-11-16T10:17:17,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/2a2a2bfd102844368a6733c4fc5eb104 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/2a2a2bfd102844368a6733c4fc5eb104 2024-11-16T10:17:17,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/884cfd2256b64e2599138fc85f5b79fa to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/884cfd2256b64e2599138fc85f5b79fa 2024-11-16T10:17:17,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/e2e97daeaa1d4babb424b4b756f8bd36 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/e2e97daeaa1d4babb424b4b756f8bd36 2024-11-16T10:17:17,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/b0fdfbd027a343df8ea3d545f421af60 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/b0fdfbd027a343df8ea3d545f421af60 2024-11-16T10:17:17,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/ddf2f811338a404c8ed9408312ad0607 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/ddf2f811338a404c8ed9408312ad0607 2024-11-16T10:17:17,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/45c8a93242284069b3d74e7dbf11bf98 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/45c8a93242284069b3d74e7dbf11bf98 2024-11-16T10:17:17,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/54de8ca623e04d488e804754217a8e9e to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/54de8ca623e04d488e804754217a8e9e 2024-11-16T10:17:17,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/aa42c993cad74408b4a33a793cd34824 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/aa42c993cad74408b4a33a793cd34824 2024-11-16T10:17:17,158 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/recovered.edits/412.seqid, newMaxSeqId=412, maxSeqId=4 2024-11-16T10:17:17,159 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. 2024-11-16T10:17:17,159 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1676): Region close journal for 571a63535798b9dbe219ba8ef9231eab: Waiting for close lock at 1731752235408Running coprocessor pre-close hooks at 1731752235408Disabling compacts and flushes for region at 1731752235408Disabling writes for close at 1731752235408Obtaining lock to block concurrent updates at 1731752235408Preparing flush snapshotting stores in 571a63535798b9dbe219ba8ef9231eab at 1731752235408Finished memstore snapshotting TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731752235409 (+1 ms)Flushing stores of TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab. at 1731752235409Flushing 571a63535798b9dbe219ba8ef9231eab/A: creating writer at 1731752235409Flushing mob file 571a63535798b9dbe219ba8ef9231eab/A: appending metadata at 1731752235416 (+7 ms)Flushing mob file 571a63535798b9dbe219ba8ef9231eab/A: closing flushed file at 1731752235416Flushing 571a63535798b9dbe219ba8ef9231eab/A: appending metadata at 1731752235831 (+415 ms)Flushing 571a63535798b9dbe219ba8ef9231eab/A: closing flushed file at 1731752235831Flushing 571a63535798b9dbe219ba8ef9231eab/B: creating writer at 1731752236240 (+409 ms)Flushing 571a63535798b9dbe219ba8ef9231eab/B: appending metadata at 1731752236241 (+1 ms)Flushing 571a63535798b9dbe219ba8ef9231eab/B: closing flushed file at 1731752236241Flushing 571a63535798b9dbe219ba8ef9231eab/C: creating writer at 1731752236654 (+413 ms)Flushing 571a63535798b9dbe219ba8ef9231eab/C: appending metadata at 1731752236658 (+4 ms)Flushing 571a63535798b9dbe219ba8ef9231eab/C: closing flushed file at 1731752236658Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3683d50e: reopening flushed file at 1731752237065 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@484d925b: reopening flushed file at 1731752237069 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@383cea7d: reopening flushed file at 1731752237072 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 571a63535798b9dbe219ba8ef9231eab in 1667ms, sequenceid=409, compaction requested=true at 1731752237075 (+3 ms)Writing region close event to WAL at 1731752237155 (+80 ms)Running coprocessor post-close hooks at 1731752237159 (+4 ms)Closed at 1731752237159 2024-11-16T10:17:17,161 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(157): Closed 571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:17,161 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=571a63535798b9dbe219ba8ef9231eab, regionState=CLOSED 2024-11-16T10:17:17,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 because future has completed 2024-11-16T10:17:17,166 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-16T10:17:17,166 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseRegionProcedure 571a63535798b9dbe219ba8ef9231eab, server=b16ff38458c1,45417,1731752048912 in 1.9080 sec 2024-11-16T10:17:17,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-11-16T10:17:17,168 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=571a63535798b9dbe219ba8ef9231eab, UNASSIGN in 1.9130 sec 2024-11-16T10:17:17,170 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-16T10:17:17,170 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9170 sec 2024-11-16T10:17:17,171 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752237171"}]},"ts":"1731752237171"} 2024-11-16T10:17:17,174 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T10:17:17,174 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T10:17:17,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9310 sec 2024-11-16T10:17:17,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T10:17:17,394 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:17:17,394 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T10:17:17,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:17:17,396 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:17:17,396 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=172, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:17:17,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-16T10:17:17,399 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:17,401 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A, FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B, FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C, FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/recovered.edits] 2024-11-16T10:17:17,403 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/3ee9d241fc7e443e99f4d3c8232f7106 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/3ee9d241fc7e443e99f4d3c8232f7106 2024-11-16T10:17:17,404 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/4a5eeb08c8044140b557690188253522 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/4a5eeb08c8044140b557690188253522 2024-11-16T10:17:17,404 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/57abf78d79604c299daed3ff1c9fb71e to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/A/57abf78d79604c299daed3ff1c9fb71e 2024-11-16T10:17:17,406 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/563b71447431461c892f0d321090fb86 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/563b71447431461c892f0d321090fb86 2024-11-16T10:17:17,406 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/9caf2c479bcf4b1fb2e6374def677678 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/9caf2c479bcf4b1fb2e6374def677678 2024-11-16T10:17:17,407 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/ac55138ba4914934b1c906c5fe8cab38 to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/B/ac55138ba4914934b1c906c5fe8cab38 2024-11-16T10:17:17,408 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/05d6ea930cf7417bb59794c36829cdea to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/05d6ea930cf7417bb59794c36829cdea 2024-11-16T10:17:17,409 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/d86f7734025642be8294b421115395bd to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/d86f7734025642be8294b421115395bd 2024-11-16T10:17:17,410 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/ea98bfb4c4614abd8b45b3dca6f6200a to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/C/ea98bfb4c4614abd8b45b3dca6f6200a 2024-11-16T10:17:17,411 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/recovered.edits/412.seqid to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab/recovered.edits/412.seqid 2024-11-16T10:17:17,412 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/default/TestAcidGuarantees/571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:17,412 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T10:17:17,412 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T10:17:17,412 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T10:17:17,414 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160ecfe9b09dcd4309ae04a53d50f6db4e_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160ecfe9b09dcd4309ae04a53d50f6db4e_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:17,415 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111619fa56339aa243b28b7274af47a9b0da_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111619fa56339aa243b28b7274af47a9b0da_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:17,416 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161d47619cd3784c47a34be52300942b43_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161d47619cd3784c47a34be52300942b43_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:17,416 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164dc4fadf8be34915994acacee5e4a504_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164dc4fadf8be34915994acacee5e4a504_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:17,417 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116625c5ce471e942358a2442cc570c11bd_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116625c5ce471e942358a2442cc570c11bd_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:17,418 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116697b2940a797457caa404fd717584d43_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116697b2940a797457caa404fd717584d43_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:17,419 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167257fba6f319458d8186a002b51f86cb_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167257fba6f319458d8186a002b51f86cb_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:17,420 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111674c48793783c4f0d8d90cef9942bf767_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111674c48793783c4f0d8d90cef9942bf767_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:17,420 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111677ef88ada296421bbf21bc46595cc0b6_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111677ef88ada296421bbf21bc46595cc0b6_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:17,421 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168063d4ec90ba468a9b98d12ae4c36f24_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168063d4ec90ba468a9b98d12ae4c36f24_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:17,422 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116841e98ff7dc04995b8daa734c1f44f92_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116841e98ff7dc04995b8daa734c1f44f92_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:17,423 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111686aa0f9aabaa4452a82beca5d9e03535_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111686aa0f9aabaa4452a82beca5d9e03535_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:17,424 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169a9a8a5dc51f47a8b1957aade4cb80bb_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169a9a8a5dc51f47a8b1957aade4cb80bb_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:17,426 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a0c5e08c21bd45deace084a9bf0294cf_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a0c5e08c21bd45deace084a9bf0294cf_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:17,427 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b831d6aaf60c443aa6f065e7bcbb5ff8_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b831d6aaf60c443aa6f065e7bcbb5ff8_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:17,428 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c0a8ada2fde7473c8538685d1cc35782_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c0a8ada2fde7473c8538685d1cc35782_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:17,429 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c90bf5b62e7246bd9c733bb68cf51786_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c90bf5b62e7246bd9c733bb68cf51786_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:17,430 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116db6b539a0f1b41d58287031077d60e8e_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116db6b539a0f1b41d58287031077d60e8e_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:17,431 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ea44957cf2b94c9d8389982a3d8f5446_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ea44957cf2b94c9d8389982a3d8f5446_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:17,431 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ee46f80885884ab7b765326a0d30956c_571a63535798b9dbe219ba8ef9231eab to hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ee46f80885884ab7b765326a0d30956c_571a63535798b9dbe219ba8ef9231eab 2024-11-16T10:17:17,432 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T10:17:17,433 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=172, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:17:17,435 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T10:17:17,437 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T10:17:17,438 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=172, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:17:17,438 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T10:17:17,438 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731752237438"}]},"ts":"9223372036854775807"} 2024-11-16T10:17:17,439 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T10:17:17,439 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 571a63535798b9dbe219ba8ef9231eab, NAME => 'TestAcidGuarantees,,1731752203707.571a63535798b9dbe219ba8ef9231eab.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T10:17:17,439 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T10:17:17,439 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731752237439"}]},"ts":"9223372036854775807"} 2024-11-16T10:17:17,440 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T10:17:17,441 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=172, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:17:17,441 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 47 msec 2024-11-16T10:17:17,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-16T10:17:17,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T10:17:17,504 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:17:17,514 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248 (was 250), OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=372 (was 369) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4149 (was 3698) - AvailableMemoryMB LEAK? - 2024-11-16T10:17:17,514 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T10:17:17,514 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T10:17:17,514 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T10:17:17,515 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:17,515 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:17,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:17:17,515 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T10:17:17,515 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=841754823, stopped=false 2024-11-16T10:17:17,515 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=b16ff38458c1,45949,1731752047888 2024-11-16T10:17:17,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T10:17:17,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45417-0x10143232da10001, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T10:17:17,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45417-0x10143232da10001, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:17:17,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:17:17,526 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T10:17:17,527 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45417-0x10143232da10001, quorum=127.0.0.1:64641, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T10:17:17,527 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T10:17:17,527 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T10:17:17,527 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:17,527 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T10:17:17,528 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b16ff38458c1,45417,1731752048912' ***** 2024-11-16T10:17:17,528 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T10:17:17,528 INFO [RS:0;b16ff38458c1:45417 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T10:17:17,528 INFO [RS:0;b16ff38458c1:45417 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T10:17:17,528 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T10:17:17,528 INFO [RS:0;b16ff38458c1:45417 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T10:17:17,528 INFO [RS:0;b16ff38458c1:45417 {}] regionserver.HRegionServer(959): stopping server b16ff38458c1,45417,1731752048912 2024-11-16T10:17:17,528 INFO [RS:0;b16ff38458c1:45417 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T10:17:17,528 INFO [RS:0;b16ff38458c1:45417 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b16ff38458c1:45417. 2024-11-16T10:17:17,528 DEBUG [RS:0;b16ff38458c1:45417 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T10:17:17,529 DEBUG [RS:0;b16ff38458c1:45417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:17,529 INFO [RS:0;b16ff38458c1:45417 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T10:17:17,529 INFO [RS:0;b16ff38458c1:45417 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T10:17:17,529 INFO [RS:0;b16ff38458c1:45417 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T10:17:17,529 INFO [RS:0;b16ff38458c1:45417 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T10:17:17,529 INFO [RS:0;b16ff38458c1:45417 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T10:17:17,529 DEBUG [RS:0;b16ff38458c1:45417 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-16T10:17:17,530 DEBUG [RS:0;b16ff38458c1:45417 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:17:17,530 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T10:17:17,530 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T10:17:17,530 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T10:17:17,530 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T10:17:17,530 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T10:17:17,530 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-16T10:17:17,552 INFO [regionserver/b16ff38458c1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T10:17:17,554 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740/.tmp/info/3ad34dc79e3247b19a3fd3c08aba541a is 95, key is TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5./info:/1731752082120/DeleteFamily/seqid=0 2024-11-16T10:17:17,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742469_1645 (size=6514) 2024-11-16T10:17:17,557 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740/.tmp/info/3ad34dc79e3247b19a3fd3c08aba541a 2024-11-16T10:17:17,579 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740/.tmp/ns/554e1a4b6bf545038fb5ca8cc8ab2f34 is 93, key is TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5./ns:/1731752082078/DeleteFamily/seqid=0 2024-11-16T10:17:17,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742470_1646 (size=5906) 2024-11-16T10:17:17,730 DEBUG [RS:0;b16ff38458c1:45417 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:17:17,930 DEBUG [RS:0;b16ff38458c1:45417 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:17:17,982 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740/.tmp/ns/554e1a4b6bf545038fb5ca8cc8ab2f34 2024-11-16T10:17:18,009 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740/.tmp/rep_barrier/88b659b8b5994f75ace061da2fd9e74f is 102, key is TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5./rep_barrier:/1731752082078/DeleteFamily/seqid=0 2024-11-16T10:17:18,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742471_1647 (size=6025) 2024-11-16T10:17:18,127 INFO [regionserver/b16ff38458c1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-16T10:17:18,127 INFO [regionserver/b16ff38458c1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-16T10:17:18,130 DEBUG [RS:0;b16ff38458c1:45417 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:17:18,249 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T10:17:18,331 DEBUG [RS:0;b16ff38458c1:45417 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:17:18,413 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740/.tmp/rep_barrier/88b659b8b5994f75ace061da2fd9e74f 2024-11-16T10:17:18,429 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740/.tmp/table/8818c197953e4fe3b4be29bcadcbd556 is 96, key is TestAcidGuarantees,,1731752053746.2bab7dc9c814fe637b384a46040e48b5./table:/1731752082078/DeleteFamily/seqid=0 2024-11-16T10:17:18,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742472_1648 (size=5936) 2024-11-16T10:17:18,531 DEBUG [RS:0;b16ff38458c1:45417 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:17:18,731 DEBUG [RS:0;b16ff38458c1:45417 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:17:18,834 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740/.tmp/table/8818c197953e4fe3b4be29bcadcbd556 2024-11-16T10:17:18,843 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740/.tmp/info/3ad34dc79e3247b19a3fd3c08aba541a as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740/info/3ad34dc79e3247b19a3fd3c08aba541a 2024-11-16T10:17:18,849 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740/info/3ad34dc79e3247b19a3fd3c08aba541a, entries=12, sequenceid=90, filesize=6.4 K 2024-11-16T10:17:18,850 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740/.tmp/ns/554e1a4b6bf545038fb5ca8cc8ab2f34 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740/ns/554e1a4b6bf545038fb5ca8cc8ab2f34 2024-11-16T10:17:18,855 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740/ns/554e1a4b6bf545038fb5ca8cc8ab2f34, entries=8, sequenceid=90, filesize=5.8 K 2024-11-16T10:17:18,856 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740/.tmp/rep_barrier/88b659b8b5994f75ace061da2fd9e74f as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740/rep_barrier/88b659b8b5994f75ace061da2fd9e74f 2024-11-16T10:17:18,861 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740/rep_barrier/88b659b8b5994f75ace061da2fd9e74f, entries=6, sequenceid=90, filesize=5.9 K 2024-11-16T10:17:18,862 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740/.tmp/table/8818c197953e4fe3b4be29bcadcbd556 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740/table/8818c197953e4fe3b4be29bcadcbd556 2024-11-16T10:17:18,866 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740/table/8818c197953e4fe3b4be29bcadcbd556, entries=7, sequenceid=90, filesize=5.8 K 2024-11-16T10:17:18,866 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1336ms, sequenceid=90, compaction requested=false 2024-11-16T10:17:18,872 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-16T10:17:18,873 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T10:17:18,873 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T10:17:18,873 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731752237529Running coprocessor pre-close hooks at 1731752237530 (+1 ms)Disabling compacts and flushes for region at 1731752237530Disabling writes for close at 1731752237530Obtaining lock to block concurrent updates at 1731752237530Preparing flush snapshotting stores in 1588230740 at 1731752237530Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731752237531 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731752237531Flushing 1588230740/info: creating writer at 1731752237531Flushing 1588230740/info: appending metadata at 1731752237553 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731752237553Flushing 1588230740/ns: creating writer at 1731752237561 (+8 ms)Flushing 1588230740/ns: appending metadata at 1731752237578 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731752237578Flushing 1588230740/rep_barrier: creating writer at 1731752237988 (+410 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731752238009 (+21 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731752238009Flushing 1588230740/table: creating writer at 1731752238416 (+407 ms)Flushing 1588230740/table: appending metadata at 1731752238429 (+13 ms)Flushing 1588230740/table: closing flushed file at 1731752238429Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cfb0482: reopening flushed file at 1731752238841 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c4edb54: reopening flushed file at 1731752238849 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42bebca5: reopening flushed file at 1731752238855 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@324ba522: reopening flushed file at 1731752238861 (+6 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1336ms, sequenceid=90, compaction requested=false at 1731752238867 (+6 ms)Writing region close event to WAL at 1731752238869 (+2 ms)Running coprocessor post-close hooks at 1731752238873 (+4 ms)Closed at 1731752238873 2024-11-16T10:17:18,873 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T10:17:18,931 INFO [RS:0;b16ff38458c1:45417 {}] regionserver.HRegionServer(976): stopping server b16ff38458c1,45417,1731752048912; all regions closed. 2024-11-16T10:17:18,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741834_1010 (size=25344) 2024-11-16T10:17:18,940 DEBUG [RS:0;b16ff38458c1:45417 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/oldWALs 2024-11-16T10:17:18,940 INFO [RS:0;b16ff38458c1:45417 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b16ff38458c1%2C45417%2C1731752048912.meta:.meta(num 1731752053290) 2024-11-16T10:17:18,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741832_1008 (size=19420748) 2024-11-16T10:17:18,945 DEBUG [RS:0;b16ff38458c1:45417 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/oldWALs 2024-11-16T10:17:18,945 INFO [RS:0;b16ff38458c1:45417 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b16ff38458c1%2C45417%2C1731752048912:(num 1731752052213) 2024-11-16T10:17:18,945 DEBUG [RS:0;b16ff38458c1:45417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:17:18,945 INFO [RS:0;b16ff38458c1:45417 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T10:17:18,945 INFO [RS:0;b16ff38458c1:45417 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T10:17:18,946 INFO [RS:0;b16ff38458c1:45417 {}] hbase.ChoreService(370): Chore service for: regionserver/b16ff38458c1:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T10:17:18,946 INFO [RS:0;b16ff38458c1:45417 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T10:17:18,946 INFO [regionserver/b16ff38458c1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T10:17:18,947 INFO [RS:0;b16ff38458c1:45417 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45417 2024-11-16T10:17:19,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45417-0x10143232da10001, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b16ff38458c1,45417,1731752048912 2024-11-16T10:17:19,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T10:17:19,064 INFO [RS:0;b16ff38458c1:45417 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T10:17:19,065 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b16ff38458c1,45417,1731752048912] 2024-11-16T10:17:19,084 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b16ff38458c1,45417,1731752048912 already deleted, retry=false 2024-11-16T10:17:19,085 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b16ff38458c1,45417,1731752048912 expired; onlineServers=0 2024-11-16T10:17:19,085 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'b16ff38458c1,45949,1731752047888' ***** 2024-11-16T10:17:19,085 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-16T10:17:19,086 INFO [M:0;b16ff38458c1:45949 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T10:17:19,086 INFO [M:0;b16ff38458c1:45949 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T10:17:19,086 DEBUG [M:0;b16ff38458c1:45949 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T10:17:19,086 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T10:17:19,086 DEBUG [M:0;b16ff38458c1:45949 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T10:17:19,087 DEBUG [master/b16ff38458c1:0:becomeActiveMaster-HFileCleaner.small.0-1731752051935 {}] cleaner.HFileCleaner(306): Exit Thread[master/b16ff38458c1:0:becomeActiveMaster-HFileCleaner.small.0-1731752051935,5,FailOnTimeoutGroup] 2024-11-16T10:17:19,087 DEBUG [master/b16ff38458c1:0:becomeActiveMaster-HFileCleaner.large.0-1731752051935 {}] cleaner.HFileCleaner(306): Exit Thread[master/b16ff38458c1:0:becomeActiveMaster-HFileCleaner.large.0-1731752051935,5,FailOnTimeoutGroup] 2024-11-16T10:17:19,087 INFO [M:0;b16ff38458c1:45949 {}] hbase.ChoreService(370): Chore service for: master/b16ff38458c1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T10:17:19,087 INFO [M:0;b16ff38458c1:45949 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T10:17:19,088 DEBUG [M:0;b16ff38458c1:45949 {}] master.HMaster(1795): Stopping service threads 2024-11-16T10:17:19,088 INFO [M:0;b16ff38458c1:45949 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T10:17:19,088 INFO [M:0;b16ff38458c1:45949 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T10:17:19,089 ERROR [M:0;b16ff38458c1:45949 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:34187 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:34187,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-16T10:17:19,090 INFO [M:0;b16ff38458c1:45949 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T10:17:19,090 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T10:17:19,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T10:17:19,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:17:19,095 DEBUG [M:0;b16ff38458c1:45949 {}] zookeeper.ZKUtil(347): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T10:17:19,095 WARN [M:0;b16ff38458c1:45949 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T10:17:19,095 INFO [M:0;b16ff38458c1:45949 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/.lastflushedseqids 2024-11-16T10:17:19,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742473_1649 (size=72) 2024-11-16T10:17:19,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45417-0x10143232da10001, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T10:17:19,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45417-0x10143232da10001, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T10:17:19,176 INFO [RS:0;b16ff38458c1:45417 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T10:17:19,176 INFO [RS:0;b16ff38458c1:45417 {}] regionserver.HRegionServer(1031): Exiting; stopping=b16ff38458c1,45417,1731752048912; zookeeper connection closed. 2024-11-16T10:17:19,177 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2bbdbf0c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2bbdbf0c 2024-11-16T10:17:19,178 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-16T10:17:19,506 INFO [M:0;b16ff38458c1:45949 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T10:17:19,506 INFO [M:0;b16ff38458c1:45949 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-16T10:17:19,506 DEBUG [M:0;b16ff38458c1:45949 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T10:17:19,506 INFO [M:0;b16ff38458c1:45949 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T10:17:19,506 DEBUG [M:0;b16ff38458c1:45949 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T10:17:19,506 DEBUG [M:0;b16ff38458c1:45949 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T10:17:19,506 DEBUG [M:0;b16ff38458c1:45949 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T10:17:19,507 INFO [M:0;b16ff38458c1:45949 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=788.86 KB heapSize=967.55 KB 2024-11-16T10:17:19,520 DEBUG [M:0;b16ff38458c1:45949 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a81fcaeac7924c0fbe617af97711f8ba is 82, key is hbase:meta,,1/info:regioninfo/1731752053369/Put/seqid=0 2024-11-16T10:17:19,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742474_1650 (size=5672) 2024-11-16T10:17:19,924 INFO [M:0;b16ff38458c1:45949 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2209 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a81fcaeac7924c0fbe617af97711f8ba 2024-11-16T10:17:19,952 DEBUG [M:0;b16ff38458c1:45949 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4bcad8b39b144e109362469f7108b375 is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x91/proc:d/1731752206779/Put/seqid=0 2024-11-16T10:17:19,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742475_1651 (size=45767) 2024-11-16T10:17:20,356 INFO [M:0;b16ff38458c1:45949 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=788.30 KB at sequenceid=2209 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4bcad8b39b144e109362469f7108b375 2024-11-16T10:17:20,360 INFO [M:0;b16ff38458c1:45949 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4bcad8b39b144e109362469f7108b375 2024-11-16T10:17:20,376 DEBUG [M:0;b16ff38458c1:45949 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d7798a32933648a285e5ebd8a77b63fb is 69, key is b16ff38458c1,45417,1731752048912/rs:state/1731752051965/Put/seqid=0 2024-11-16T10:17:20,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073742476_1652 (size=5156) 2024-11-16T10:17:20,779 INFO [M:0;b16ff38458c1:45949 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2209 (bloomFilter=true), to=hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d7798a32933648a285e5ebd8a77b63fb 2024-11-16T10:17:20,782 DEBUG [M:0;b16ff38458c1:45949 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a81fcaeac7924c0fbe617af97711f8ba as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a81fcaeac7924c0fbe617af97711f8ba 2024-11-16T10:17:20,785 INFO [M:0;b16ff38458c1:45949 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a81fcaeac7924c0fbe617af97711f8ba, entries=8, sequenceid=2209, filesize=5.5 K 2024-11-16T10:17:20,785 DEBUG [M:0;b16ff38458c1:45949 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4bcad8b39b144e109362469f7108b375 as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4bcad8b39b144e109362469f7108b375 2024-11-16T10:17:20,788 INFO [M:0;b16ff38458c1:45949 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4bcad8b39b144e109362469f7108b375 2024-11-16T10:17:20,788 INFO [M:0;b16ff38458c1:45949 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4bcad8b39b144e109362469f7108b375, entries=172, sequenceid=2209, filesize=44.7 K 2024-11-16T10:17:20,788 DEBUG [M:0;b16ff38458c1:45949 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d7798a32933648a285e5ebd8a77b63fb as hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d7798a32933648a285e5ebd8a77b63fb 2024-11-16T10:17:20,790 INFO [M:0;b16ff38458c1:45949 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34187/user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d7798a32933648a285e5ebd8a77b63fb, entries=1, sequenceid=2209, filesize=5.0 K 2024-11-16T10:17:20,791 INFO [M:0;b16ff38458c1:45949 {}] regionserver.HRegion(3140): Finished flush of dataSize ~788.86 KB/807793, heapSize ~967.26 KB/990472, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1285ms, sequenceid=2209, compaction requested=false 2024-11-16T10:17:20,792 INFO [M:0;b16ff38458c1:45949 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T10:17:20,792 DEBUG [M:0;b16ff38458c1:45949 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731752239506Disabling compacts and flushes for region at 1731752239506Disabling writes for close at 1731752239506Obtaining lock to block concurrent updates at 1731752239507 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731752239507Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=807793, getHeapSize=990712, getOffHeapSize=0, getCellsCount=2406 at 1731752239507Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731752239507Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731752239507Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731752239520 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731752239520Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731752239931 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731752239951 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731752239951Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731752240360 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731752240375 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731752240375Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fd82149: reopening flushed file at 1731752240782 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5dc7a307: reopening flushed file at 1731752240785 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6366e5cb: reopening flushed file at 1731752240788 (+3 ms)Finished flush of dataSize ~788.86 KB/807793, heapSize ~967.26 KB/990472, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1285ms, sequenceid=2209, compaction requested=false at 1731752240791 (+3 ms)Writing region close event to WAL at 1731752240792 (+1 ms)Closed at 1731752240792 2024-11-16T10:17:20,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44191 is added to blk_1073741830_1006 (size=952585) 2024-11-16T10:17:20,796 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/4619d6c5-cb27-11ad-e3d2-d3aa72c3f9d0/MasterData/WALs/b16ff38458c1,45949,1731752047888/b16ff38458c1%2C45949%2C1731752047888.1731752051225 not finished, retry = 0 2024-11-16T10:17:20,897 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T10:17:20,897 INFO [M:0;b16ff38458c1:45949 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T10:17:20,897 INFO [M:0;b16ff38458c1:45949 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45949 2024-11-16T10:17:20,898 INFO [M:0;b16ff38458c1:45949 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T10:17:21,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T10:17:21,072 INFO [M:0;b16ff38458c1:45949 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T10:17:21,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45949-0x10143232da10000, quorum=127.0.0.1:64641, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T10:17:21,180 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T10:17:21,182 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T10:17:21,182 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T10:17:21,182 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T10:17:21,182 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/hadoop.log.dir/,STOPPED} 2024-11-16T10:17:21,185 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T10:17:21,185 WARN [BP-1732300688-172.17.0.2-1731752043743 heartbeating to localhost/127.0.0.1:34187 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T10:17:21,185 WARN [BP-1732300688-172.17.0.2-1731752043743 heartbeating to localhost/127.0.0.1:34187 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1732300688-172.17.0.2-1731752043743 (Datanode Uuid c61bbee8-8e42-49b2-9ad9-00f98f01cef0) service to localhost/127.0.0.1:34187 2024-11-16T10:17:21,185 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T10:17:21,188 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/cluster_bb61cbda-8873-24b8-881e-9662d491e561/data/data1/current/BP-1732300688-172.17.0.2-1731752043743 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T10:17:21,188 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/cluster_bb61cbda-8873-24b8-881e-9662d491e561/data/data2/current/BP-1732300688-172.17.0.2-1731752043743 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T10:17:21,188 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T10:17:21,194 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T10:17:21,195 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T10:17:21,195 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T10:17:21,195 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T10:17:21,195 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/415342d7-d24f-32eb-6313-a5a770af3569/hadoop.log.dir/,STOPPED} 2024-11-16T10:17:21,210 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-16T10:17:21,345 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down